diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
| commit | c4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch) | |
| tree | 81ce5675ce886aad0309a41f4d6a696826e0a061 /bench/bench_tokenizer.c | |
| parent | b377ba8656bfb807ed8dda6c9f26942b50ce626d (diff) | |
| download | ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip | |
chore: run clang-format
Diffstat (limited to 'bench/bench_tokenizer.c')
| -rw-r--r-- | bench/bench_tokenizer.c | 63 |
1 files changed, 32 insertions, 31 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index 1033f2e..fb6d37a 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -10,24 +10,23 @@ int qwen_verbose = 0; -static double now_ms(void) -{ +static double now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0; } -static int same_ids(const int *a, int na, const int *b, int nb) -{ - if (na != nb) return 0; +static int same_ids(const int *a, int na, const int *b, int nb) { + if (na != nb) + return 0; for (int i = 0; i < na; i++) { - if (a[i] != b[i]) return 0; + if (a[i] != b[i]) + return 0; } return 1; } -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { int status = 1; int **gold = NULL; int *gold_n = NULL; @@ -41,7 +40,8 @@ int main(int argc, char **argv) const char *vocab = argv[1]; int runs = atoi(argv[2]); - if (runs <= 0) return 2; + if (runs <= 0) + return 2; int n_texts = argc - 3; char **texts = argv + 3; @@ -56,17 +56,18 @@ int main(int argc, char **argv) gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n)); bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs)); caps = (int *)calloc((size_t)n_texts, sizeof(*caps)); - if (!gold || !gold_n || !bufs || !caps) goto cleanup; + if (!gold || !gold_n || !bufs || !caps) + goto cleanup; int tokens_per_pass = 0; for (int i = 0; i < n_texts; i++) { gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]); - if (!gold[i] || gold_n[i] <= 0) goto cleanup; + if (!gold[i] || gold_n[i] <= 0) + goto cleanup; tokens_per_pass += gold_n[i]; int n_ws = 0; - int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], - &n_ws); + int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws); if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) { free(ids_ws); goto cleanup; @@ -74,12 +75,13 @@ int main(int argc, char **argv) free(ids_ws); caps[i] = (int)(strlen(texts[i]) * 2 + 8); - if (caps[i] < gold_n[i]) caps[i] = gold_n[i]; + if (caps[i] < gold_n[i]) + caps[i] = gold_n[i]; bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int)); - if (!bufs[i]) goto cleanup; + if (!bufs[i]) + goto cleanup; int n_into = 0; - if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], - &n_into) != 0 || + if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 || !same_ids(gold[i], gold_n[i], bufs[i], n_into)) goto cleanup; } @@ -105,8 +107,7 @@ int main(int argc, char **argv) for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], - &n); + int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n); if (!ids || n != gold_n[i]) { free(ids); goto cleanup; @@ -122,30 +123,30 @@ int main(int argc, char **argv) for (int r = 0; r < runs; r++) { for (int i = 0; i < n_texts; i++) { int n = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], - caps[i], &n); - if (rc != 0 || n != gold_n[i]) goto cleanup; + int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n); + if (rc != 0 || n != gold_n[i]) + goto cleanup; into_tokens += n; } } double into_ms = now_ms() - t0; - printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, - tokens_per_pass); + printf("texts=%d runs=%d tokens/pass=%d\n", n_texts, runs, tokens_per_pass); printf("old_malloc: %.3f ms %.0f tokens/s\n", old_ms, (double)old_tokens / (old_ms / 1000.0)); - printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", - ws_ms, (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms); - printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", - into_ms, (double)into_tokens / (into_ms / 1000.0), - old_ms / into_ms); + printf("workspace_malloc: %.3f ms %.0f tokens/s speedup=%.2fx\n", ws_ms, + (double)ws_tokens / (ws_ms / 1000.0), old_ms / ws_ms); + printf("workspace_into: %.3f ms %.0f tokens/s speedup=%.2fx\n", into_ms, + (double)into_tokens / (into_ms / 1000.0), old_ms / into_ms); status = 0; cleanup: for (int i = 0; i < n_texts; i++) { - if (gold) free(gold[i]); - if (bufs) free(bufs[i]); + if (gold) + free(gold[i]); + if (bufs) + free(bufs[i]); } free(gold); free(gold_n); |
