diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-13 02:39:21 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:10 +0200 |
| commit | b98609527907655bbd0dac1f966002c163f77cc0 (patch) | |
| tree | 8f704cbc96bb880b7a6a94c7b76da38bc20d6480 | |
| parent | 81b6e93083d950ba63ee677ceb33b432cb1bf2cd (diff) | |
| download | ffwd-b98609527907655bbd0dac1f966002c163f77cc0.tar.gz ffwd-b98609527907655bbd0dac1f966002c163f77cc0.zip | |
chore: fix static analyzer warnings in tests/bench
| -rw-r--r-- | bench/bench_tokenizer.c | 51 | ||||
| -rw-r--r-- | tests/test_kernels.c | 5 | ||||
| -rw-r--r-- | tests/test_server.c | 13 |
3 files changed, 47 insertions, 22 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c index 425a5fc..1033f2e 100644 --- a/bench/bench_tokenizer.c +++ b/bench/bench_tokenizer.c @@ -28,6 +28,12 @@ static int same_ids(const int *a, int na, const int *b, int nb) int main(int argc, char **argv) { + int status = 1; + int **gold = NULL; + int *gold_n = NULL; + int **bufs = NULL; + int *caps = NULL; + if (argc < 4) { fprintf(stderr, "usage: %s VOCAB_JSON RUNS TEXT...\n", argv[0]); return 2; @@ -43,37 +49,39 @@ int main(int argc, char **argv) qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); if (!tok || !ws) { fprintf(stderr, "failed to initialize tokenizer\n"); - return 1; + goto cleanup; } - int **gold = (int **)calloc((size_t)n_texts, sizeof(*gold)); - int *gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n)); - int **bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs)); - int *caps = (int *)calloc((size_t)n_texts, sizeof(*caps)); - if (!gold || !gold_n || !bufs || !caps) return 1; + gold = (int **)calloc((size_t)n_texts, sizeof(*gold)); + gold_n = (int *)calloc((size_t)n_texts, sizeof(*gold_n)); + bufs = (int **)calloc((size_t)n_texts, sizeof(*bufs)); + caps = (int *)calloc((size_t)n_texts, sizeof(*caps)); + if (!gold || !gold_n || !bufs || !caps) goto cleanup; int tokens_per_pass = 0; for (int i = 0; i < n_texts; i++) { gold[i] = qwen_tokenizer_encode(tok, texts[i], &gold_n[i]); - if (!gold[i] || gold_n[i] <= 0) return 1; + if (!gold[i] || gold_n[i] <= 0) goto cleanup; tokens_per_pass += gold_n[i]; int n_ws = 0; int *ids_ws = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n_ws); - if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) - return 1; + if (!ids_ws || !same_ids(gold[i], gold_n[i], ids_ws, n_ws)) { + free(ids_ws); + goto cleanup; + } free(ids_ws); caps[i] = (int)(strlen(texts[i]) * 2 + 8); if (caps[i] < gold_n[i]) caps[i] = gold_n[i]; bufs[i] = (int *)malloc((size_t)caps[i] * sizeof(int)); - if (!bufs[i]) return 1; + if (!bufs[i]) goto cleanup; int n_into = 0; if (qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n_into) != 0 || !same_ids(gold[i], gold_n[i], bufs[i], n_into)) - return 1; + goto cleanup; } double t0 = now_ms(); @@ -82,7 +90,10 @@ int main(int argc, char **argv) for (int i = 0; i < n_texts; i++) { int n = 0; int *ids = qwen_tokenizer_encode(tok, texts[i], &n); - if (!ids || n != gold_n[i]) return 1; + if (!ids || n != gold_n[i]) { + free(ids); + goto cleanup; + } old_tokens += n; free(ids); } @@ -96,7 +107,10 @@ int main(int argc, char **argv) int n = 0; int *ids = qwen_tokenizer_encode_with_workspace(tok, ws, texts[i], &n); - if (!ids || n != gold_n[i]) return 1; + if (!ids || n != gold_n[i]) { + free(ids); + goto cleanup; + } ws_tokens += n; free(ids); } @@ -110,7 +124,7 @@ int main(int argc, char **argv) int n = 0; int rc = qwen_tokenizer_encode_into(tok, ws, texts[i], bufs[i], caps[i], &n); - if (rc != 0 || n != gold_n[i]) return 1; + if (rc != 0 || n != gold_n[i]) goto cleanup; into_tokens += n; } } @@ -126,9 +140,12 @@ int main(int argc, char **argv) into_ms, (double)into_tokens / (into_ms / 1000.0), old_ms / into_ms); + status = 0; + +cleanup: for (int i = 0; i < n_texts; i++) { - free(gold[i]); - free(bufs[i]); + if (gold) free(gold[i]); + if (bufs) free(bufs[i]); } free(gold); free(gold_n); @@ -136,5 +153,5 @@ int main(int argc, char **argv) free(caps); qwen_tokenizer_workspace_free(ws); qwen_tokenizer_free(tok); - return 0; + return status; } diff --git a/tests/test_kernels.c b/tests/test_kernels.c index a390411..208dd02 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -365,7 +365,10 @@ static void test_bf16_widen_buf(void) { qwen_bf16_to_f32_buf(got, src, (size_t)n); for (int i = 0; i < n; i++) { float want = bf16_to_f32(src[i]); - if (memcmp(&got[i], &want, sizeof(float)) != 0) { + uint32_t got_bits, want_bits; + memcpy(&got_bits, &got[i], sizeof(got_bits)); + memcpy(&want_bits, &want, sizeof(want_bits)); + if (got_bits != want_bits) { fprintf(stderr, "bf16_widen n=%d at %d: %.9g want %.9g\n", n, i, got[i], want); failures++; diff --git a/tests/test_server.c b/tests/test_server.c index ba933c7..2e99881 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -677,11 +677,16 @@ static void test_http_rerank(int port) ? cJSON_GetObjectItemCaseSensitive(usage, "document_tokens") : NULL; cJSON *total_tokens = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; - TEST_ASSERT(query_tokens && query_tokens->valueint == 32); - TEST_ASSERT(document_tokens && document_tokens->valueint > 0); - TEST_ASSERT(total_tokens && - total_tokens->valueint == + TEST_ASSERT(query_tokens && document_tokens && total_tokens); + if (query_tokens && document_tokens && total_tokens) { + TEST_ASSERT(cJSON_IsNumber(query_tokens)); + TEST_ASSERT(cJSON_IsNumber(document_tokens)); + TEST_ASSERT(cJSON_IsNumber(total_tokens)); + TEST_ASSERT(query_tokens->valueint == 32); + TEST_ASSERT(document_tokens->valueint > 0); + TEST_ASSERT(total_tokens->valueint == query_tokens->valueint + document_tokens->valueint); + } cJSON_Delete(root); } free(body); |
