aboutsummaryrefslogtreecommitdiff
path: root/bench/bench_tokenizer.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
commitc4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch)
tree81ce5675ce886aad0309a41f4d6a696826e0a061 /bench/bench_tokenizer.c
parentb377ba8656bfb807ed8dda6c9f26942b50ce626d (diff)
downloadffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz
ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip
chore: run clang-format
Diffstat (limited to 'bench/bench_tokenizer.c')
-rw-r--r--bench/bench_tokenizer.c63
1 files changed, 32 insertions, 31 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 1033f2e..fb6d37a 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -10,24 +10,23 @@
int qwen_verbose = 0;
-static double now_ms(void)
-{
+static double now_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
}
-static int same_ids(const int *a, int na, const int *b, int nb)
-{
- if (na != nb) return 0;
+static int same_ids(const int *a, int na, const int *b, int nb) {
+ if (na != nb)
+ return 0;
for (int i = 0; i < na; i++) {
- if (a[i] != b[i]) return 0;
+ if (a[i] != b[i])
+ return 0;
}
return 1;
}
-int main(int argc, char **argv)
-{
+int main(int argc, char **argv) {
int status = 1;
int **gold = NULL;
int *gold_n = NULL;
@@ -41,7 +40,8 @@ int main(int argc, char **argv)
const char *vocab = argv[1];
int runs = atoi(argv[2]);
- if (runs <= 0) return 2;
+ if (runs <= 0)
+ return 2;
int n_texts = argc - 3;
char **texts = argv + 3;
@@ -56,17 +56,18 @@ int main(int argc, char **argv)
gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
- if (!gold || !gold_n || !bufs || !caps) goto cleanup;
+ if (!gold || !gold_n || !bufs || !caps)
+ goto cleanup;
int tokens_per_pass = 0;
for (int i = 0; i < n_texts; i++) {
gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]);
- if (!gold[i] || gold_n[i] <= 0) goto cleanup;
+ if (!gold[i] || gold_n[i] <= 0)
+ goto cleanup;
tokens_per_pass += gold_n[i];
int n_ws = 0;
- int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
- &n_ws);
+ int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws);
if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) {
free(ids_ws);
goto cleanup;
@@ -74,12 +75,13 @@ int main(int argc, char **argv)
free(ids_ws);
caps[i] = (int)(strlen(texts[i]) * 2 + 8);
- if (caps[i] < gold_n[i]) caps[i] = gold_n[i];
+ if (caps[i] < gold_n[i])
+ caps[i] = gold_n[i];
bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int));
- if (!bufs[i]) goto cleanup;
+ if (!bufs[i])
+ goto cleanup;
int n_into = 0;
- if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i],
- &n_into) != 0 ||
+ if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 ||
!same_ids(gold[i], gold_n[i], bufs[i], n_into))
goto cleanup;
}
@@ -105,8 +107,7 @@ int main(int argc, char **argv)
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
- &n);
+ int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n);
if (!ids || n != gold_n[i]) {
free(ids);
goto cleanup;
@@ -122,30 +123,30 @@ int main(int argc, char **argv)
for (int r = 0; r < runs; r++) {
for (int i = 0; i < n_texts; i++) {
int n = 0;
- int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i],
- caps[i], &n);
- if (rc != 0 || n != gold_n[i]) goto cleanup;
+ int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n);
+ if (rc != 0 || n != gold_n[i])
+ goto cleanup;
into_tokens += n;
}
}
double into_ms = now_ms() - t0;
- printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs,
- tokens_per_pass);
+ printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, tokens_per_pass);
printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms,
(double)old_tokens / (old_ms / 1000.0));
- printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n",
- ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms);
- printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n",
- into_ms, (double)into_tokens / (into_ms / 1000.0),
- old_ms / into_ms);
+ printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", ws_ms,
+ (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms);
+ printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", into_ms,
+ (double)into_tokens / (into_ms / 1000.0), old_ms / into_ms);
status = 0;
cleanup:
for (int i = 0; i < n_texts; i++) {
- if (gold) free(gold[i]);
- if (bufs) free(bufs[i]);
+ if (gold)
+ free(gold[i]);
+ if (bufs)
+ free(bufs[i]);
}
free(gold);
free(gold_n);