aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-13 02:39:21 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:10 +0200
commitb98609527907655bbd0dac1f966002c163f77cc0 (patch)
tree8f704cbc96bb880b7a6a94c7b76da38bc20d6480
parent81b6e93083d950ba63ee677ceb33b432cb1bf2cd (diff)
downloadffwd-b98609527907655bbd0dac1f966002c163f77cc0.tar.gz
ffwd-b98609527907655bbd0dac1f966002c163f77cc0.zip
chore: fix static analyzer warnings in tests/bench
-rw-r--r--bench/bench_tokenizer.c51
-rw-r--r--tests/test_kernels.c5
-rw-r--r--tests/test_server.c13
3 files changed, 47 insertions, 22 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 425a5fc..1033f2e 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -28,6 +28,12 @@ static int same_ids(const int *a, int na, const int *b, int nb)
int main(int argc, char **argv)
{
+ int status = 1;
+ int **gold = NULL;
+ int *gold_n = NULL;
+ int **bufs = NULL;
+ int *caps = NULL;
+
if (argc < 4) {
fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]);
return 2;
@@ -43,37 +49,39 @@ int main(int argc, char **argv)
qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new();
if (!tok || !ws) {
fprintf(stderr, "failed to initialize tokenizer\n");
- return 1;
+ goto cleanup;
}
- int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold));
- int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
- int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
- int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
- if (!gold || !gold_n || !bufs || !caps) return 1;
+ gold = (int **)calloc((size_t)n_texts, sizeof(*gold));
+ gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n));
+ bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs));
+ caps = (int *)calloc((size_t)n_texts, sizeof(*caps));
+ if (!gold || !gold_n || !bufs || !caps) goto cleanup;
int tokens_per_pass = 0;
for (int i = 0; i < n_texts; i++) {
gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]);
- if (!gold[i] || gold_n[i] <= 0) return 1;
+ if (!gold[i] || gold_n[i] <= 0) goto cleanup;
tokens_per_pass += gold_n[i];
int n_ws = 0;
int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
&n_ws);
- if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws))
- return 1;
+ if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) {
+ free(ids_ws);
+ goto cleanup;
+ }
free(ids_ws);
caps[i] = (int)(strlen(texts[i]) * 2 + 8);
if (caps[i] < gold_n[i]) caps[i] = gold_n[i];
bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int));
- if (!bufs[i]) return 1;
+ if (!bufs[i]) goto cleanup;
int n_into = 0;
if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i],
&n_into) != 0 ||
!same_ids(gold[i], gold_n[i], bufs[i], n_into))
- return 1;
+ goto cleanup;
}
double t0 = now_ms();
@@ -82,7 +90,10 @@ int main(int argc, char **argv)
for (int i = 0; i < n_texts; i++) {
int n = 0;
int *ids = qwen_tokenizer_encode(tok, texts[i], &n);
- if (!ids || n != gold_n[i]) return 1;
+ if (!ids || n != gold_n[i]) {
+ free(ids);
+ goto cleanup;
+ }
old_tokens += n;
free(ids);
}
@@ -96,7 +107,10 @@ int main(int argc, char **argv)
int n = 0;
int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i],
&n);
- if (!ids || n != gold_n[i]) return 1;
+ if (!ids || n != gold_n[i]) {
+ free(ids);
+ goto cleanup;
+ }
ws_tokens += n;
free(ids);
}
@@ -110,7 +124,7 @@ int main(int argc, char **argv)
int n = 0;
int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i],
caps[i], &n);
- if (rc != 0 || n != gold_n[i]) return 1;
+ if (rc != 0 || n != gold_n[i]) goto cleanup;
into_tokens += n;
}
}
@@ -126,9 +140,12 @@ int main(int argc, char **argv)
into_ms, (double)into_tokens / (into_ms / 1000.0),
old_ms / into_ms);
+ status = 0;
+
+cleanup:
for (int i = 0; i < n_texts; i++) {
- free(gold[i]);
- free(bufs[i]);
+ if (gold) free(gold[i]);
+ if (bufs) free(bufs[i]);
}
free(gold);
free(gold_n);
@@ -136,5 +153,5 @@ int main(int argc, char **argv)
free(caps);
qwen_tokenizer_workspace_free(ws);
qwen_tokenizer_free(tok);
- return 0;
+ return status;
}
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index a390411..208dd02 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -365,7 +365,10 @@ static void test_bf16_widen_buf(void) {
qwen_bf16_to_f32_buf(got, src, (size_t)n);
for (int i = 0; i < n; i++) {
float want = bf16_to_f32(src[i]);
- if (memcmp(&got[i], &want, sizeof(float)) != 0) {
+ uint32_t got_bits, want_bits;
+ memcpy(&got_bits, &got[i], sizeof(got_bits));
+ memcpy(&want_bits, &want, sizeof(want_bits));
+ if (got_bits != want_bits) {
fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n",
n, i, got[i], want);
failures++;
diff --git a/tests/test_server.c b/tests/test_server.c
index ba933c7..2e99881 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -677,11 +677,16 @@ static void test_http_rerank(int port)
? cJSON_GetObjectItemCaseSensitive(usage, "document_tokens") : NULL;
cJSON *total_tokens = usage
? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
- TEST_ASSERT(query_tokens && query_tokens->valueint == 32);
- TEST_ASSERT(document_tokens && document_tokens->valueint > 0);
- TEST_ASSERT(total_tokens &&
- total_tokens->valueint ==
+ TEST_ASSERT(query_tokens && document_tokens && total_tokens);
+ if (query_tokens && document_tokens && total_tokens) {
+ TEST_ASSERT(cJSON_IsNumber(query_tokens));
+ TEST_ASSERT(cJSON_IsNumber(document_tokens));
+ TEST_ASSERT(cJSON_IsNumber(total_tokens));
+ TEST_ASSERT(query_tokens->valueint == 32);
+ TEST_ASSERT(document_tokens->valueint > 0);
+ TEST_ASSERT(total_tokens->valueint ==
query_tokens->valueint + document_tokens->valueint);
+ }
cJSON_Delete(root);
}
free(body);