diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:45:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | bc8bebe78358e3a37bad0867b2d94955cac01d47 (patch) | |
| tree | d5a49a34b95b45cec0917a2a07707159825fb912 /bench/bench_tokenizer.c | |
| parent | 6652fcbc653d84e75cb5c08c5f5edf52263ed50f (diff) | |
| download | ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.tar.gz ffwd-bc8bebe78358e3a37bad0867b2d94955cac01d47.zip | |
bench: add e2e benchmarks + new API compat fixes
Diffstat (limited to 'bench/bench_tokenizer.c')
| -rw-r--r-- | bench/bench_tokenizer.c | 63 |
1 files changed, 30 insertions, 33 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index 8175ffe..9bb8822 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -1,9 +1,9 @@ /* bench/bench_tokenizer.c - tokenizer encode-path benchmark. * Build via `make bench-tokenizer`; accepts a model dir or vocab.json. */ -#include "tokenizer_bpe.h" -#include "tokenizer_sentencepiece.h" -#include "tokenizer_wordpiece.h" +#include "bpe.h" +#include "sentencepiece.h" +#include "wordpiece.h" #include <stdio.h> #include <stdlib.h> @@ -11,8 +11,6 @@ #include <unistd.h> #include <time.h> -int embed_verbose = 0; - static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); @@ -50,8 +48,8 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) { snprintf(p, sizeof(p), "%s/vocab.txt", path); if (path_exists(p)) { bt->kind = TOK_WORDPIECE; - bt->tok = wordpiece_tokenizer_load(path); - bt->ws = wordpiece_workspace_new(); + bt->tok = tok_wp_load(path); + bt->ws = tok_wp_workspace_new(); return bt->tok && bt->ws ? 0 : -1; } snprintf(p, sizeof(p), "%s/sentencepiece.bpe.model", path); @@ -62,8 +60,8 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) { } if (path_exists(p)) { bt->kind = TOK_SENTENCEPIECE; - bt->tok = sentencepiece_tokenizer_load(path); - bt->ws = sentencepiece_workspace_new(); + bt->tok = tok_spm_load(path); + bt->ws = tok_spm_workspace_new(); return bt->tok && bt->ws ? 0 : -1; } bt->kind = TOK_BPE; @@ -71,64 +69,63 @@ static int load_tokenizer(bench_tok_t *bt, const char *path) { snprintf(p, sizeof(p), "%s/vocab.json", path); if (path_exists(p)) vocab = p; - bt->tok = embed_tokenizer_load(vocab); - bt->ws = embed_tokenizer_workspace_new(); + bt->tok = tok_bpe_load(vocab); + bt->ws = tok_bpe_workspace_new(); return bt->tok && bt->ws ? 0 : -1; } static int *tok_encode(bench_tok_t *bt, const char *text, int *n) { switch (bt->kind) { case TOK_WORDPIECE: - return wordpiece_tokenizer_encode((wordpiece_tokenizer_t *)bt->tok, text, n); + return tok_wp_encode((tok_wp_t *)bt->tok, text, n); case TOK_SENTENCEPIECE: - return sentencepiece_tokenizer_encode((sentencepiece_tokenizer_t *)bt->tok, text, n); + return tok_spm_encode((tok_spm_t *)bt->tok, text, n); default: - return embed_tokenizer_encode((embed_tokenizer_t *)bt->tok, text, n); + return tok_bpe_encode((tok_bpe_t *)bt->tok, text, n); } } static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) { switch (bt->kind) { case TOK_WORDPIECE: - return wordpiece_tokenizer_encode_with_workspace((wordpiece_tokenizer_t *)bt->tok, - (wordpiece_workspace_t *)bt->ws, text, n); + return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok, + (tok_wp_workspace_t *)bt->ws, text, n); case TOK_SENTENCEPIECE: - return sentencepiece_tokenizer_encode_with_workspace( - (sentencepiece_tokenizer_t *)bt->tok, (sentencepiece_workspace_t *)bt->ws, text, n); + return tok_spm_encode_with_workspace( + (tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, n); default: - return embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)bt->tok, - (embed_tokenizer_workspace_t *)bt->ws, text, n); + return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok, + (tok_bpe_workspace_t *)bt->ws, text, n); } } static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) { switch (bt->kind) { case TOK_WORDPIECE: - return wordpiece_tokenizer_encode_into((wordpiece_tokenizer_t *)bt->tok, - (wordpiece_workspace_t *)bt->ws, text, out, cap, n); + return tok_wp_encode_into((tok_wp_t *)bt->tok, + (tok_wp_workspace_t *)bt->ws, text, out, cap, n); case TOK_SENTENCEPIECE: - return sentencepiece_tokenizer_encode_into((sentencepiece_tokenizer_t *)bt->tok, - (sentencepiece_workspace_t *)bt->ws, text, out, - cap, n); + return tok_spm_encode_into((tok_spm_t *)bt->tok, + (tok_spm_workspace_t *)bt->ws, text, out, cap, n); default: - return embed_tokenizer_encode_into((embed_tokenizer_t *)bt->tok, - (embed_tokenizer_workspace_t *)bt->ws, text, out, cap, n); + return tok_bpe_encode_into((tok_bpe_t *)bt->tok, + (tok_bpe_workspace_t *)bt->ws, text, out, cap, n); } } static void free_tokenizer(bench_tok_t *bt) { switch (bt->kind) { case TOK_WORDPIECE: - wordpiece_workspace_free((wordpiece_workspace_t *)bt->ws); - wordpiece_tokenizer_free((wordpiece_tokenizer_t *)bt->tok); + tok_wp_workspace_free((tok_wp_workspace_t *)bt->ws); + tok_wp_free((tok_wp_t *)bt->tok); break; case TOK_SENTENCEPIECE: - sentencepiece_workspace_free((sentencepiece_workspace_t *)bt->ws); - sentencepiece_tokenizer_free((sentencepiece_tokenizer_t *)bt->tok); + tok_spm_workspace_free((tok_spm_workspace_t *)bt->ws); + tok_spm_free((tok_spm_t *)bt->tok); break; default: - embed_tokenizer_workspace_free((embed_tokenizer_workspace_t *)bt->ws); - embed_tokenizer_free((embed_tokenizer_t *)bt->tok); + tok_bpe_workspace_free((tok_bpe_workspace_t *)bt->ws); + tok_bpe_free((tok_bpe_t *)bt->tok); break; } } |
