aboutsummaryrefslogtreecommitdiff
path: root/bench
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-13 02:39:21 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:10 +0200
commitb98609527907655bbd0dac1f966002c163f77cc0 (patch)
tree8f704cbc96bb880b7a6a94c7b76da38bc20d6480 /bench
parent81b6e93083d950ba63ee677ceb33b432cb1bf2cd (diff)
downloadffwd-b98609527907655bbd0dac1f966002c163f77cc0.tar.gz
ffwd-b98609527907655bbd0dac1f966002c163f77cc0.zip
chore: fix static analyzer warnings in tests/bench
Diffstat (limited to 'bench')
-rw-r--r--bench/bench_tokenizer.c51
1 files changed, 34 insertions, 17 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 425a5fc..1033f2e 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -28,6 +28,12 @@ static int same_ids(const int *a, int na, const int *b, int nb)
int main(int argc, char **argv)
{
+ int status = 1;
+ int **gold = NULL;
+ int *gold_n = NULL;
+ int **bufs = NULL;
+ int *caps = NULL;
+
if (argc < 4) {
fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]);
return 2;
@@ -43,37 +49,39 @@ int main(int argc, char **argv)
qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new();
if (!tok || !ws) {
fprintf(stderr, "failed to initialize tokenizer\n");
- return 1;
+ goto cleanup;
}
- int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold));
- int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
- int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
- int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
- if (!gold || !gold_n || !bufs || !caps) return 1;
+ gold = (int **)calloc((size_t)n_texts, sizeof(*gold));
+ gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
+ bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
+ caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
+ if (!gold || !gold_n || !bufs || !caps) goto cleanup;
int tokens_per_pass = 0;
for (int i = 0; i < n_texts; i++) {
gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]);
- if (!gold[i] || gold_n[i] <= 0) return 1;
+ if (!gold[i] || gold_n[i] <= 0) goto cleanup;
tokens_per_pass += gold_n[i];
int n_ws = 0;
int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
&n_ws);
- if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws))
- return 1;
+ if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) {
+ free(ids_ws);
+ goto cleanup;
+ }
free(ids_ws);
caps[i] = (int)(strlen(texts[i]) * 2 + 8);
if (caps[i] < gold_n[i]) caps[i] = gold_n[i];
bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int));
- if (!bufs[i]) return 1;
+ if (!bufs[i]) goto cleanup;
int n_into = 0;
if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i],
&n_into) != 0 ||
!same_ids(gold[i], gold_n[i], bufs[i], n_into))
- return 1;
+ goto cleanup;
}
double t0 = now_ms();
@@ -82,7 +90,10 @@ int main(int argc, char **argv)
for (int i = 0; i < n_texts; i++) {
int n = 0;
int *ids = qwen_tokenizer_encode(tok, texts[i], &n);
- if (!ids || n != gold_n[i]) return 1;
+ if (!ids || n != gold_n[i]) {
+ free(ids);
+ goto cleanup;
+ }
old_tokens += n;
free(ids);
}
@@ -96,7 +107,10 @@ int main(int argc, char **argv)
int n = 0;
int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
&n);
- if (!ids || n != gold_n[i]) return 1;
+ if (!ids || n != gold_n[i]) {
+ free(ids);
+ goto cleanup;
+ }
ws_tokens += n;
free(ids);
}
@@ -110,7 +124,7 @@ int main(int argc, char **argv)
int n = 0;
int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i],
caps[i], &n);
- if (rc != 0 || n != gold_n[i]) return 1;
+ if (rc != 0 || n != gold_n[i]) goto cleanup;
into_tokens += n;
}
}
@@ -126,9 +140,12 @@ int main(int argc, char **argv)
into_ms, (double)into_tokens / (into_ms / 1000.0),
old_ms / into_ms);
+ status = 0;
+
+cleanup:
for (int i = 0; i < n_texts; i++) {
- free(gold[i]);
- free(bufs[i]);
+ if (gold) free(gold[i]);
+ if (bufs) free(bufs[i]);
}
free(gold);
free(gold_n);
@@ -136,5 +153,5 @@ int main(int argc, char **argv)
free(caps);
qwen_tokenizer_workspace_free(ws);
qwen_tokenizer_free(tok);
- return 0;
+ return status;
}