diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /tests | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/test_kernels.c | 96 | ||||
| -rw-r--r-- | tests/test_late.c | 12 | ||||
| -rw-r--r-- | tests/test_safetensors.c | 2 | ||||
| -rw-r--r-- | tests/test_server.c | 30 | ||||
| -rw-r--r-- | tests/test_tokenizer.c | 122 | ||||
| -rw-r--r-- | tests/test_wordpiece.c | 2 | ||||
| -rw-r--r-- | tests/test_workspace.c | 14 |
7 files changed, 139 insertions, 139 deletions
diff --git a/tests/test_kernels.c b/tests/test_kernels.c index 4a9e9db..4742b70 100644 --- a/tests/test_kernels.c +++ b/tests/test_kernels.c @@ -2,8 +2,8 @@ * Runs via `make test` (generic and BLAS variants) and * scripts/check_kernel_golden.py. No model files required. */ -#include "qwen_kernels.h" -#include "qwen_kernels_impl.h" +#include "kernels.h" +#include "kernels_impl.h" #include <math.h> #include <stdint.h> @@ -12,7 +12,7 @@ #include <string.h> static int failures = 0; -int qwen_verbose = 0; +int embed_verbose = 0; static void expect_close(const char *name, float got, float want, float tol) { float diff = fabsf(got - want); @@ -42,7 +42,7 @@ static void test_rms_norm(void) { const float w[4] = {1.0f, 0.5f, -1.0f, 2.0f}; float out[8]; - qwen_rms_norm(out, x, w, seq, hidden, 1e-6f); + embed_rms_norm(out, x, w, seq, hidden, 1e-6f); for (int s = 0; s < seq; s++) { float ss = 0.0f; @@ -65,7 +65,7 @@ static void test_layer_norm(void) { const float b[4] = {0.1f, -0.2f, 0.3f, -0.4f}; float out[8]; - qwen_layer_norm(out, x, g, b, seq, hidden, 1e-12f); + embed_layer_norm(out, x, g, b, seq, hidden, 1e-12f); for (int s = 0; s < seq; s++) { float mean = 0.0f; @@ -88,7 +88,7 @@ static void test_layer_norm(void) { /* In-place (out aliases x) must match the out-of-place result. */ float z[8]; memcpy(z, x, sizeof(z)); - qwen_layer_norm(z, z, g, b, seq, hidden, 1e-12f); + embed_layer_norm(z, z, g, b, seq, hidden, 1e-12f); for (int i = 0; i < 8; i++) expect_close("layer_norm_inplace", z[i], out[i], 1e-6f); } @@ -97,7 +97,7 @@ static void test_gelu(void) { float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; float out[5]; memcpy(out, x, sizeof(out)); - qwen_gelu_inplace(out, 5); + embed_gelu_inplace(out, 5); for (int i = 0; i < 5; i++) { float want = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f)); expect_close("gelu", out[i], want, 1e-6f); @@ -112,7 +112,7 @@ static void test_gelu_tanh(void) { float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f}; float out[5]; memcpy(out, x, sizeof(out)); - qwen_gelu_tanh_inplace(out, 5); + embed_gelu_tanh_inplace(out, 5); for (int i = 0; i < 5; i++) { float v = x[i]; float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v); @@ -134,8 +134,8 @@ static void test_rope_neox(void) { float orig[8]; memcpy(orig, x, sizeof(x)); - qwen_compute_rope_neox(cosv, sinv, positions, seq, head_dim, 10000.0f); - qwen_apply_rope_neox(x, cosv, sinv, seq, heads, head_dim); + embed_compute_rope_neox(cosv, sinv, positions, seq, head_dim, 10000.0f); + embed_apply_rope_neox(x, cosv, sinv, seq, heads, head_dim); for (int s = 0; s < seq; s++) { for (int d = 0; d < head_dim / 2; d++) { @@ -225,7 +225,7 @@ static void test_packed_gqa_attention(void) { V[i] = sinf((float)i * 0.07f) - cosf((float)i * 0.13f); } - qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, + embed_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 0); @@ -254,12 +254,12 @@ static void test_packed_gqa_attention_long(void) { V[i] = sinf((float)i * 0.007f) - cosf((float)i * 0.013f); } - qwen_set_threads(4); - qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, + embed_set_threads(4); + embed_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 0); - qwen_set_threads(1); + embed_set_threads(1); for (int i = 0; i < total * q_hidden; i++) expect_close("packed_gqa_long", got[i], want[i], 2e-5f); @@ -290,12 +290,12 @@ static void test_packed_causal_gqa_attention(void) { V[i] = sinf((float)i * 0.009f) - cosf((float)i * 0.015f); } - qwen_set_threads(4); - qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, + embed_set_threads(4); + embed_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim)); reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim, 1.0f / sqrtf((float)head_dim), 1); - qwen_set_threads(1); + embed_set_threads(1); for (int i = 0; i < total * q_hidden; i++) expect_close("packed_causal_gqa", got[i], want[i], 2e-5f); @@ -315,7 +315,7 @@ static void test_bf16_linear(void) { for (int i = 0; i < 6; i++) wbf16[i] = f32_to_bf16(wf32[i]); - qwen_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim); + embed_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < out_dim; o++) { @@ -351,8 +351,8 @@ static void check_bf16_qkv(int n_threads) { wv[i] = f32_to_bf16(wv_f32[i]); } - qwen_set_threads(n_threads); - qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim); + embed_set_threads(n_threads); + embed_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < q_dim; o++) { @@ -376,7 +376,7 @@ static void check_bf16_qkv(int n_threads) { static void test_bf16_qkv(void) { check_bf16_qkv(1); check_bf16_qkv(3); - qwen_set_threads(1); + embed_set_threads(1); } /* seq==1 routes through bf16_matvec_threaded; a pool splits the output @@ -392,8 +392,8 @@ static void check_bf16_matvec(int n_threads) { for (int i = 0; i < out_dim * in_dim; i++) w[i] = f32_to_bf16(cosf((float)i * 0.21f) + 0.05f); - qwen_set_threads(n_threads); - qwen_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim); + embed_set_threads(n_threads); + embed_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim); for (int o = 0; o < out_dim; o++) { float want = 0.0f; @@ -406,10 +406,10 @@ static void check_bf16_matvec(int n_threads) { static void test_bf16_matvec(void) { check_bf16_matvec(1); check_bf16_matvec(4); - qwen_set_threads(1); + embed_set_threads(1); } -/* Above QWEN_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to +/* Above EMBED_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to * rms_norm_worker; row-wise math is identical, so results must match the * single-thread run exactly. */ static void test_rms_norm_threaded(void) { @@ -436,11 +436,11 @@ static void test_rms_norm_threaded(void) { for (int i = 0; i < hidden; i++) w[i] = 0.8f + 0.01f * (float)(i % 7); - qwen_set_threads(1); - qwen_rms_norm(one, x, w, seq, hidden, 1e-6f); - qwen_set_threads(4); - qwen_rms_norm(many, x, w, seq, hidden, 1e-6f); - qwen_set_threads(1); + embed_set_threads(1); + embed_rms_norm(one, x, w, seq, hidden, 1e-6f); + embed_set_threads(4); + embed_rms_norm(many, x, w, seq, hidden, 1e-6f); + embed_set_threads(1); for (size_t i = 0; i < elems; i++) { if (one[i] != many[i]) { @@ -469,7 +469,7 @@ static void test_bf16_widen_buf(void) { for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) { int n = sizes[t]; memset(got, 0, sizeof(got)); - qwen_bf16_to_f32_buf(got, src, (size_t)n); + embed_bf16_to_f32_buf(got, src, (size_t)n); for (int i = 0; i < n; i++) { float want = bf16_to_f32(src[i]); uint32_t got_bits, want_bits; @@ -503,8 +503,8 @@ static void check_bf16_pair(int n_threads) { wb[i] = f32_to_bf16(wb_f32[i]); } - qwen_set_threads(n_threads); - qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim); + embed_set_threads(n_threads); + embed_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim); for (int s = 0; s < seq; s++) { for (int o = 0; o < a_dim; o++) { @@ -525,10 +525,10 @@ static void check_bf16_pair(int n_threads) { static void test_bf16_pair(void) { check_bf16_pair(1); check_bf16_pair(4); - qwen_set_threads(1); + embed_set_threads(1); } -/* The dispatch macros in qwen_kernels_impl.h pick the SIMD variant at +/* The dispatch macros in embed_kernels_impl.h pick the SIMD variant at * compile time, so on arm64/x86 the generic C kernels are linked but never * called. Run them directly against the selected implementation on * remainder-lane-hostile sizes; on a plain build impl == generic and the @@ -554,46 +554,46 @@ static void test_generic_vs_impl(void) { } float yg[OUT], yi[OUT]; - qwen_bf16_matvec_fused_generic(yg, x, W, bias, IN, OUT); - qwen_bf16_matvec_fused_impl(yi, x, W, bias, IN, OUT); + embed_bf16_matvec_fused_generic(yg, x, W, bias, IN, OUT); + embed_bf16_matvec_fused_impl(yi, x, W, bias, IN, OUT); for (int o = 0; o < OUT; o++) expect_close("matvec_fused generic-vs-impl", yg[o], yi[o], 1e-5f); - qwen_bf16_matvec_fused_generic(yg, x, W, NULL, IN, OUT); - qwen_bf16_matvec_fused_impl(yi, x, W, NULL, IN, OUT); + embed_bf16_matvec_fused_generic(yg, x, W, NULL, IN, OUT); + embed_bf16_matvec_fused_impl(yi, x, W, NULL, IN, OUT); for (int o = 0; o < OUT; o++) expect_close("matvec_fused nobias generic-vs-impl", yg[o], yi[o], 1e-5f); const int sizes[] = {1, 7, 64, NMAX}; for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) { int n = sizes[t]; - expect_close("dot generic-vs-impl", qwen_dot_f32_generic(a, b, n), - qwen_dot_f32_impl(a, b, n), 1e-5f); + expect_close("dot generic-vs-impl", embed_dot_f32_generic(a, b, n), + embed_dot_f32_impl(a, b, n), 1e-5f); memcpy(d0, a, (size_t)n * sizeof(float)); memcpy(d1, a, (size_t)n * sizeof(float)); - qwen_vec_scale_inplace_generic(d0, 1.37f, n); - qwen_vec_scale_inplace_impl(d1, 1.37f, n); + embed_vec_scale_inplace_generic(d0, 1.37f, n); + embed_vec_scale_inplace_impl(d1, 1.37f, n); for (int i = 0; i < n; i++) expect_close("scale generic-vs-impl", d0[i], d1[i], 1e-6f); memcpy(d0, a, (size_t)n * sizeof(float)); memcpy(d1, a, (size_t)n * sizeof(float)); - qwen_vec_axpy_inplace_generic(d0, b, -0.61f, n); - qwen_vec_axpy_inplace_impl(d1, b, -0.61f, n); + embed_vec_axpy_inplace_generic(d0, b, -0.61f, n); + embed_vec_axpy_inplace_impl(d1, b, -0.61f, n); for (int i = 0; i < n; i++) expect_close("axpy generic-vs-impl", d0[i], d1[i], 1e-6f); memcpy(d0, a, (size_t)n * sizeof(float)); memcpy(d1, a, (size_t)n * sizeof(float)); - qwen_vec_scale_add_generic(d0, b, 0.83f, n); - qwen_vec_scale_add_impl(d1, b, 0.83f, n); + embed_vec_scale_add_generic(d0, b, 0.83f, n); + embed_vec_scale_add_impl(d1, b, 0.83f, n); for (int i = 0; i < n; i++) expect_close("scale_add generic-vs-impl", d0[i], d1[i], 1e-6f); } } int main(void) { - qwen_set_threads(1); + embed_set_threads(1); test_rms_norm(); test_layer_norm(); test_gelu(); diff --git a/tests/test_late.c b/tests/test_late.c index 661661c..ec6c03f 100644 --- a/tests/test_late.c +++ b/tests/test_late.c @@ -3,7 +3,7 @@ * byte-complete tokenizer fixture. Runs via `make test`. */ #include "embed.h" -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tiny_model.h" #include "tok_fixture.h" @@ -63,7 +63,7 @@ int main(void) { tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE}; int hidden = dims.hidden; - qwen_tokenizer_t *tok = NULL; + embed_tokenizer_t *tok = NULL; embed_model_t *base = NULL; embed_workspace_t *base_ws = NULL; embed_late_model_t *late = NULL; @@ -145,14 +145,14 @@ int main(void) { char vocab_path[1280]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); - tok = qwen_tokenizer_load(vocab_path); + tok = embed_tokenizer_load(vocab_path); if (!tok) { fprintf(stderr, "tokenizer load failed\n"); goto fail; } int n_tokens = 0; - ids = qwen_tokenizer_encode(tok, "hello world", &n_tokens); + ids = embed_tokenizer_encode(tok, "hello world", &n_tokens); if (!ids || n_tokens <= 1) { fprintf(stderr, "tokenization failed\n"); goto fail; @@ -244,7 +244,7 @@ int main(void) { int lens[3] = {0}; int offsets[4] = {0}; for (int i = 0; i < 3; i++) { - doc_ids[i] = qwen_tokenizer_encode(tok, doc_texts[i], &lens[i]); + doc_ids[i] = embed_tokenizer_encode(tok, doc_texts[i], &lens[i]); if (!doc_ids[i] || lens[i] <= 0) { fprintf(stderr, "doc tokenization failed\n"); goto fail; @@ -401,7 +401,7 @@ fail: free(raw); free(states); free(ids); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_late_workspace_free(ws); embed_late_model_free(late); embed_workspace_free(base_ws); diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c index 36ec8c5..ccfd6fb 100644 --- a/tests/test_safetensors.c +++ b/tests/test_safetensors.c @@ -2,7 +2,7 @@ * Hermetic: synthesizes every fixture in a temp directory. Runs via * `make test`. */ -#include "qwen_safetensors.h" +#include "safetensors.h" #include <stdint.h> #include <stdio.h> diff --git a/tests/test_server.c b/tests/test_server.c index 9ebb8f2..c0c85f1 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -6,7 +6,7 @@ * path over loopback sockets. Runs via `make test`. */ #define EMBED_SERVER_TEST -#include "../embed_server.c" +#include "../src/server.c" #include "tiny_model.h" #include "tok_fixture.h" @@ -596,7 +596,7 @@ static void test_http_embeddings(int port) { "{\"model\":\"Qwen3-Embedding-0.6B\"," "\"input\":\"hello\",\"encoding_format\":\"float\"}", NULL, &body) == 200); - cJSON *qwen_full = body ? parse_embeddings_body(body, 1) : NULL; + cJSON *embed_full = body ? parse_embeddings_body(body, 1) : NULL; free(body); body = NULL; TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, @@ -604,10 +604,10 @@ static void test_http_embeddings(int port) { "\"input\":\"hello\",\"dimensions\":32," "\"encoding_format\":\"float\"}", NULL, &body) == 200); - cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL; - if (qwen_full && qwen_mrl) { - cJSON *full_data = cJSON_GetObjectItemCaseSensitive(qwen_full, "data"); - cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data"); + cJSON *embed_mrl = body ? parse_embeddings_body(body, 1) : NULL; + if (embed_full && embed_mrl) { + cJSON *full_data = cJSON_GetObjectItemCaseSensitive(embed_full, "data"); + cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(embed_mrl, "data"); cJSON *full_emb = cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(full_data, 0), "embedding"); cJSON *mrl_emb = @@ -622,12 +622,12 @@ static void test_http_embeddings(int port) { prefix_diff = d; } TEST_ASSERT(prefix_diff > 0.0); - cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage"); + cJSON *usage = cJSON_GetObjectItemCaseSensitive(embed_mrl, "usage"); cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; TEST_ASSERT(total && total->valueint == 2); } - cJSON_Delete(qwen_full); - cJSON_Delete(qwen_mrl); + cJSON_Delete(embed_full); + cJSON_Delete(embed_mrl); free(body); body = NULL; @@ -949,7 +949,7 @@ static void test_http_rerank(int port) { } static void test_http_server(void) { - char dir[1024], late_dir[1024], qwen_dir[1024], gte_dir[1024]; + char dir[1024], late_dir[1024], embed_dir[1024], gte_dir[1024]; snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); if (!mkdtemp(dir)) { @@ -964,9 +964,9 @@ static void test_http_server(void) { test_failures++; return; } - snprintf(qwen_dir, sizeof(qwen_dir), "%s/embed-srv-qwen-test-XXXXXX", + snprintf(embed_dir, sizeof(embed_dir), "%s/embed-srv-qwen-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(qwen_dir)) { + if (!mkdtemp(embed_dir)) { fprintf(stderr, "FAIL: Qwen mkdtemp\n"); test_failures++; return; @@ -985,8 +985,8 @@ static void test_http_server(void) { if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 || tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 || tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 || - tf_write_vocab(qwen_dir) != 0 || - tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0 || + tf_write_vocab(embed_dir) != 0 || + tm_write_qwen3_model_dims(embed_dir, "F32", &dims, TF_EOT_ID) != 0 || tf_write_vocab(gte_dir) != 0 || tm_write_qwen2_model_dims(gte_dir, "F32", >e_dims, TF_EOT_ID, 0) != 0) { fprintf(stderr, "FAIL: fixture write\n"); @@ -1008,7 +1008,7 @@ static void test_http_server(void) { ctx.spec[2].id = "pplx-embed-v1-late-0.6b"; ctx.spec[2].path = late_dir; ctx.spec[3].id = "Qwen3-Embedding-0.6B"; - ctx.spec[3].path = qwen_dir; + ctx.spec[3].path = embed_dir; ctx.spec[4].id = "gte-Qwen2-1.5B-instruct"; ctx.spec[4].path = gte_dir; ctx.cfg.models = ctx.spec; diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 5920977..3ef4280 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -4,7 +4,7 @@ * integer-pair fast path, and that dropping the integer tables (forcing the * string-based fallback) produces identical ids. Runs via `make test`. */ -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tok_fixture.h" #include <stdio.h> @@ -14,7 +14,7 @@ #include <unistd.h> static int failures = 0; -int qwen_verbose = 0; +int embed_verbose = 0; #define TEST_ASSERT(cond) \ do { \ @@ -24,14 +24,14 @@ int qwen_verbose = 0; } \ } while (0) -static void check_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_ids(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, const int *want, int n_want, const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL); if (!ids) return; @@ -52,19 +52,19 @@ static void check_ids(const qwen_tokenizer_t *tok, /* Concatenated decode of an encoding must reproduce the (NFC-normalized) * input - byte-level BPE is lossless. */ -static void check_roundtrip(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_roundtrip(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *text, const char *want, const char *label) { int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n); + int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n); TEST_ASSERT(ids != NULL && n > 0); if (!ids) return; size_t cap = 1; for (int i = 0; i < n; i++) - cap += strlen(qwen_tokenizer_decode(tok, ids[i])); + cap += strlen(embed_tokenizer_decode(tok, ids[i])); char *got = (char *)malloc(cap); if (!got) { free(ids); @@ -73,7 +73,7 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, } got[0] = '\0'; for (int i = 0; i < n; i++) - strcat(got, qwen_tokenizer_decode(tok, ids[i])); + strcat(got, embed_tokenizer_decode(tok, ids[i])); if (strcmp(got, want) != 0) { fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got, want); @@ -84,18 +84,18 @@ static void check_roundtrip(const qwen_tokenizer_t *tok, } /* Two spellings of the same text must encode identically. */ -static void check_same_ids(const qwen_tokenizer_t *tok, - qwen_tokenizer_workspace_t *ws, +static void check_same_ids(const embed_tokenizer_t *tok, + embed_tokenizer_workspace_t *ws, const char *a, const char *b, const char *label) { int na = 0, nb = 0; - int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na); + int *ia = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, a, &na); if (!ia) { TEST_ASSERT(0); return; } - int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb); + int *ib = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, b, &nb); if (!ib) { free(ia); TEST_ASSERT(0); @@ -110,7 +110,7 @@ static void check_same_ids(const qwen_tokenizer_t *tok, } static void -run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) { +run_cases(const embed_tokenizer_t *tok, embed_tokenizer_workspace_t *ws, const char *label) { /* h e l l o -> he -> he,ll -> hell -> hello */ const int hello[] = {259}; check_ids(tok, ws, "hello", hello, 1, label); @@ -212,7 +212,7 @@ static void write_file(const char *dir, const char *name, const char *body) { static void test_loader_edges(const char *root) { char dir[1152], path[2048]; - TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL); + TEST_ASSERT(embed_tokenizer_load("/nonexistent/vocab.json") == NULL); /* Not a JSON object. */ snprintf(dir, sizeof(dir), "%s/badjson", root); @@ -223,7 +223,7 @@ static void test_loader_edges(const char *root) { write_file(dir, "vocab.json", "[1, 2]\n"); write_file(dir, "merges.txt", "a b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - TEST_ASSERT(qwen_tokenizer_load(path) == NULL); + TEST_ASSERT(embed_tokenizer_load(path) == NULL); /* Missing or pair-less merges.txt is tolerated: the tokenizer loads * and encoding falls back to byte/vocab-level (no merges applied). */ @@ -234,16 +234,16 @@ static void test_loader_edges(const char *root) { } write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *nom = qwen_tokenizer_load(path); + embed_tokenizer_t *nom = embed_tokenizer_load(path); TEST_ASSERT(nom != NULL); if (nom) { - qwen_tokenizer_workspace_t *nws = qwen_tokenizer_workspace_new(); + embed_tokenizer_workspace_t *nws = embed_tokenizer_workspace_new(); int n = 0; - int *ids = qwen_tokenizer_encode_with_workspace(nom, nws, "ab", &n); + int *ids = embed_tokenizer_encode_with_workspace(nom, nws, "ab", &n); TEST_ASSERT(ids && n == 2 && ids[0] == 0 && ids[1] == 1); free(ids); - qwen_tokenizer_workspace_free(nws); - qwen_tokenizer_free(nom); + embed_tokenizer_workspace_free(nws); + embed_tokenizer_free(nom); } snprintf(dir, sizeof(dir), "%s/emptymerges", root); @@ -254,9 +254,9 @@ static void test_loader_edges(const char *root) { write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n"); write_file(dir, "merges.txt", "#version: 0.2\n\n \n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *emp = qwen_tokenizer_load(path); + embed_tokenizer_t *emp = embed_tokenizer_load(path); TEST_ASSERT(emp != NULL); - qwen_tokenizer_free(emp); + embed_tokenizer_free(emp); /* Escape forms in vocab keys; a malformed merge line is skipped. */ snprintf(dir, sizeof(dir), "%s/escapes", root); @@ -270,28 +270,28 @@ static void test_loader_edges(const char *root) { " \"s\\\\t\":6, \"p\\/q\":7, \"u\\tv\":8, \"\\u0041Z\":9}\n"); write_file(dir, "merges.txt", "#version: 0.2\nmalformed-line\na b\n"); snprintf(path, sizeof(path), "%s/vocab.json", dir); - qwen_tokenizer_t *tok = qwen_tokenizer_load(path); + embed_tokenizer_t *tok = embed_tokenizer_load(path); TEST_ASSERT(tok != NULL); if (tok) { - TEST_ASSERT(qwen_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "x\ny") == 4); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "q\"r") == 5); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "s\\t") == 6); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "p/q") == 7); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "u\tv") == 8); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "AZ") == 9); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 2), "ab") == 0); + TEST_ASSERT(embed_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3); + TEST_ASSERT(embed_tokenizer_token_id(tok, "x\ny") == 4); + TEST_ASSERT(embed_tokenizer_token_id(tok, "q\"r") == 5); + TEST_ASSERT(embed_tokenizer_token_id(tok, "s\\t") == 6); + TEST_ASSERT(embed_tokenizer_token_id(tok, "p/q") == 7); + TEST_ASSERT(embed_tokenizer_token_id(tok, "u\tv") == 8); + TEST_ASSERT(embed_tokenizer_token_id(tok, "AZ") == 9); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 2), "ab") == 0); /* Keys outside the GPT-2 byte alphabet decode to replacement. */ - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 3), "?") == 0); - qwen_tokenizer_free(tok); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 3), "?") == 0); + embed_tokenizer_free(tok); } /* A bare filename exercises the no-directory merges-path branch. */ char saved_cwd[1024]; if (getcwd(saved_cwd, sizeof(saved_cwd)) && chdir(dir) == 0) { - qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json"); + embed_tokenizer_t *rel = embed_tokenizer_load("vocab.json"); TEST_ASSERT(rel != NULL); - qwen_tokenizer_free(rel); + embed_tokenizer_free(rel); if (chdir(saved_cwd) != 0) { perror("chdir back"); exit(2); @@ -310,7 +310,7 @@ int main(void) { char vocab_path[2048]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path); TEST_ASSERT(tok != NULL); if (!tok) return 1; @@ -319,48 +319,48 @@ int main(void) { TEST_ASSERT(tok->int_merges != NULL); TEST_ASSERT(tok->cp_to_id != NULL); - qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new(); + embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new(); TEST_ASSERT(ws != NULL); run_cases(tok, ws, "fast"); /* decode: byte tokens, merged tokens, and out-of-range ids. */ - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 'a'), "a") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 259), "hello") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, -1), "") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); - TEST_ASSERT(strcmp(qwen_tokenizer_decode(NULL, 0), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 'a'), "a") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 259), "hello") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, -1), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0); + TEST_ASSERT(strcmp(embed_tokenizer_decode(NULL, 0), "") == 0); /* token_id lookups. */ - TEST_ASSERT(qwen_tokenizer_token_id(tok, "hello") == 259); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); - TEST_ASSERT(qwen_tokenizer_token_id(tok, "no-such-token") == -1); - TEST_ASSERT(qwen_tokenizer_token_id(NULL, "a") == -1); - TEST_ASSERT(qwen_tokenizer_token_id(tok, NULL) == -1); + TEST_ASSERT(embed_tokenizer_token_id(tok, "hello") == 259); + TEST_ASSERT(embed_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID); + TEST_ASSERT(embed_tokenizer_token_id(tok, "no-such-token") == -1); + TEST_ASSERT(embed_tokenizer_token_id(NULL, "a") == -1); + TEST_ASSERT(embed_tokenizer_token_id(tok, NULL) == -1); /* encode_into: exact fit, truncation (-2 plus the untruncated prefix), * zero capacity, empty text, and argument guards. */ { int out[8]; int n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262); /* On overflow the prefix is written and n reports the total * required count, so a caller can re-size and retry. */ n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3); TEST_ASSERT(out[0] == 259 && out[1] == 263); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3); n = -1; - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); - TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); - TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0); + TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1); + TEST_ASSERT(embed_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1); } - TEST_ASSERT(qwen_tokenizer_encode(tok, "", NULL) == NULL); - TEST_ASSERT(qwen_tokenizer_encode(NULL, "x", NULL) == NULL); - qwen_tokenizer_workspace_free(NULL); - qwen_tokenizer_free(NULL); + TEST_ASSERT(embed_tokenizer_encode(tok, "", NULL) == NULL); + TEST_ASSERT(embed_tokenizer_encode(NULL, "x", NULL) == NULL); + embed_tokenizer_workspace_free(NULL); + embed_tokenizer_free(NULL); /* Drop the integer tables: the string-based fallback must produce the * exact same ids. */ @@ -371,8 +371,8 @@ int main(void) { tok->cp_to_id = NULL; run_cases(tok, ws, "slow"); - qwen_tokenizer_workspace_free(ws); - qwen_tokenizer_free(tok); + embed_tokenizer_workspace_free(ws); + embed_tokenizer_free(tok); test_loader_edges(dir); diff --git a/tests/test_wordpiece.c b/tests/test_wordpiece.c index 1fd6ff8..d40b7d2 100644 --- a/tests/test_wordpiece.c +++ b/tests/test_wordpiece.c @@ -3,7 +3,7 @@ * punctuation splitting, CJK char-splitting, special-token resolution, and the * workspace encode paths. No model files or network needed. */ -#include "wordpiece_tokenizer.h" +#include "tokenizer_wordpiece.h" #include <stdio.h> #include <stdlib.h> diff --git a/tests/test_workspace.c b/tests/test_workspace.c index a906689..c8b6ee8 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -4,7 +4,7 @@ * what scripts/check_workspace_api.py does. Runs via `make test`. */ #include "embed.h" -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tiny_model.h" #include "tok_fixture.h" @@ -199,10 +199,10 @@ int main(int argc, char **argv) { embed_workspace_t *batch_ws = embed_workspace_new(model); embed_workspace_t *single_ws = embed_workspace_new(model); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path); if (!batch_ws || !single_ws || !tok) { fprintf(stderr, "failed to allocate workspace or tokenizer\n"); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); @@ -227,7 +227,7 @@ int main(int argc, char **argv) { } for (int i = 0; i < batch; i++) { - ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]); + ids[i] = embed_tokenizer_encode(tok, texts[i], &ntok[i]); if (!ids[i] || ntok[i] <= 0) { fprintf(stderr, "failed to tokenize input %d\n", i); goto fail; @@ -331,7 +331,7 @@ int main(int argc, char **argv) { /* Same separator resolution as the server: the fixture vocab defines * <|endoftext|>, real snapshots use the reserved id. */ - int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>"); + int sep_id = embed_tokenizer_token_id(tok, "<|endoftext|>"); if (sep_id < 0) sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID; if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) { @@ -349,7 +349,7 @@ int main(int argc, char **argv) { for (int i = 0; i < batch; i++) free(ids[i]); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); @@ -367,7 +367,7 @@ int main(int argc, char **argv) { fail: for (int i = 0; i < batch; i++) free(ids[i]); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); |
