diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 08:48:04 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | a960ebb8dec645ab619a68d7412d4392e99f4b7a (patch) | |
| tree | 06ef707b7ab7b7ff4064ec7acf94bd4a4cd92409 /bench | |
| parent | 809c26869b83530d91a5068e5085da7fccbf5399 (diff) | |
| download | ffwd-a960ebb8dec645ab619a68d7412d4392e99f4b7a.tar.gz ffwd-a960ebb8dec645ab619a68d7412d4392e99f4b7a.zip | |
tests,bench: move generated C checkers into standing files
Diffstat (limited to 'bench')
| -rw-r--r-- | bench/bench_tokenizer.c | 140 |
1 files changed, 140 insertions, 0 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c new file mode 100644 index 0000000..425a5fc --- /dev/null +++ b/bench/bench_tokenizer.c @@ -0,0 +1,140 @@ +/* bench/bench_tokenizer.c - tokenizer encode-path benchmark. + * Build via `make bench-tokenizer`; needs a model dir (vocab.json). */ + +#include "qwen_tokenizer.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +int qwen_verbose = 0; + +static double now_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; +} + +static int same_ids(const int *a, int na, const int *b, int nb) +{ + if (na != nb) return 0; + for (int i = 0; i < na; i++) { + if (a[i] != b[i]) return 0; + } + return 1; +} + +int main(int argc, char **argv) +{ + if (argc < 4) { + fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]); + return 2; + } + + const char *vocab = argv[1]; + int runs = atoi(argv[2]); + if (runs <= 0) return 2; + int n_texts = argc - 3; + char **texts = argv + 3; + + qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab); + qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); + if (!tok || !ws) { + fprintf(stderr, "failed to initialize tokenizer\n"); + return 1; + } + + int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold)); + int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n)); + int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs)); + int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps)); + if (!gold || !gold_n || !bufs || !caps) return 1; + + int tokens_per_pass = 0; + for (int i = 0; i < n_texts; i++) { + gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]); + if (!gold[i] || gold_n[i] <= 0) return 1; + tokens_per_pass += gold_n[i]; + + int n_ws = 0; + int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], + &n_ws); + if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) + return 1; + free(ids_ws); + + caps[i] = (int)(strlen(texts[i]) * 2 + 8); + if (caps[i] < gold_n[i]) caps[i] = gold_n[i]; + bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int)); + if (!bufs[i]) return 1; + int n_into = 0; + if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], + &n_into) != 0 || + !same_ids(gold[i], gold_n[i], bufs[i], n_into)) + return 1; + } + + double t0 = now_ms(); + long long old_tokens = 0; + for (int r = 0; r < runs; r++) { + for (int i = 0; i < n_texts; i++) { + int n = 0; + int *ids = qwen_tokenizer_encode(tok, texts[i], &n); + if (!ids || n != gold_n[i]) return 1; + old_tokens += n; + free(ids); + } + } + double old_ms = now_ms() - t0; + + t0 = now_ms(); + long long ws_tokens = 0; + for (int r = 0; r < runs; r++) { + for (int i = 0; i < n_texts; i++) { + int n = 0; + int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], + &n); + if (!ids || n != gold_n[i]) return 1; + ws_tokens += n; + free(ids); + } + } + double ws_ms = now_ms() - t0; + + t0 = now_ms(); + long long into_tokens = 0; + for (int r = 0; r < runs; r++) { + for (int i = 0; i < n_texts; i++) { + int n = 0; + int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], + caps[i], &n); + if (rc != 0 || n != gold_n[i]) return 1; + into_tokens += n; + } + } + double into_ms = now_ms() - t0; + + printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, + tokens_per_pass); + printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms, + (double)old_tokens / (old_ms / 1000.0)); + printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", + ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms); + printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", + into_ms, (double)into_tokens / (into_ms / 1000.0), + old_ms / into_ms); + + for (int i = 0; i < n_texts; i++) { + free(gold[i]); + free(bufs[i]); + } + free(gold); + free(gold_n); + free(bufs); + free(caps); + qwen_tokenizer_workspace_free(ws); + qwen_tokenizer_free(tok); + return 0; +} |
