aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
commitd1ba1ebc06ef0d62ca932388fae83e6992594719 (patch)
treef962207b779016593258f990348216999a58766a /tests
parentbe20dfda0f836808dccfc14db244592e41ba28bc (diff)
downloadffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz
ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'tests')
-rw-r--r--tests/test_kernels.c96
-rw-r--r--tests/test_late.c12
-rw-r--r--tests/test_safetensors.c2
-rw-r--r--tests/test_server.c30
-rw-r--r--tests/test_tokenizer.c122
-rw-r--r--tests/test_wordpiece.c2
-rw-r--r--tests/test_workspace.c14
7 files changed, 139 insertions, 139 deletions
diff --git a/tests/test_kernels.c b/tests/test_kernels.c
index 4a9e9db..4742b70 100644
--- a/tests/test_kernels.c
+++ b/tests/test_kernels.c
@@ -2,8 +2,8 @@
* Runs via `make test` (generic and BLAS variants) and
* scripts/check_kernel_golden.py. No model files required. */
-#include "qwen_kernels.h"
-#include "qwen_kernels_impl.h"
+#include "kernels.h"
+#include "kernels_impl.h"
#include <math.h>
#include <stdint.h>
@@ -12,7 +12,7 @@
#include <string.h>
static int failures = 0;
-int qwen_verbose = 0;
+int embed_verbose = 0;
static void expect_close(const char *name, float got, float want, float tol) {
float diff = fabsf(got - want);
@@ -42,7 +42,7 @@ static void test_rms_norm(void) {
const float w[4] = {1.0f, 0.5f, -1.0f, 2.0f};
float out[8];
- qwen_rms_norm(out, x, w, seq, hidden, 1e-6f);
+ embed_rms_norm(out, x, w, seq, hidden, 1e-6f);
for (int s = 0; s < seq; s++) {
float ss = 0.0f;
@@ -65,7 +65,7 @@ static void test_layer_norm(void) {
const float b[4] = {0.1f, -0.2f, 0.3f, -0.4f};
float out[8];
- qwen_layer_norm(out, x, g, b, seq, hidden, 1e-12f);
+ embed_layer_norm(out, x, g, b, seq, hidden, 1e-12f);
for (int s = 0; s < seq; s++) {
float mean = 0.0f;
@@ -88,7 +88,7 @@ static void test_layer_norm(void) {
/* In-place (out aliases x) must match the out-of-place result. */
float z[8];
memcpy(z, x, sizeof(z));
- qwen_layer_norm(z, z, g, b, seq, hidden, 1e-12f);
+ embed_layer_norm(z, z, g, b, seq, hidden, 1e-12f);
for (int i = 0; i < 8; i++)
expect_close("layer_norm_inplace", z[i], out[i], 1e-6f);
}
@@ -97,7 +97,7 @@ static void test_gelu(void) {
float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f};
float out[5];
memcpy(out, x, sizeof(out));
- qwen_gelu_inplace(out, 5);
+ embed_gelu_inplace(out, 5);
for (int i = 0; i < 5; i++) {
float want = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
expect_close("gelu", out[i], want, 1e-6f);
@@ -112,7 +112,7 @@ static void test_gelu_tanh(void) {
float x[5] = {0.0f, 1.0f, -1.0f, 2.5f, -3.0f};
float out[5];
memcpy(out, x, sizeof(out));
- qwen_gelu_tanh_inplace(out, 5);
+ embed_gelu_tanh_inplace(out, 5);
for (int i = 0; i < 5; i++) {
float v = x[i];
float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
@@ -134,8 +134,8 @@ static void test_rope_neox(void) {
float orig[8];
memcpy(orig, x, sizeof(x));
- qwen_compute_rope_neox(cosv, sinv, positions, seq, head_dim, 10000.0f);
- qwen_apply_rope_neox(x, cosv, sinv, seq, heads, head_dim);
+ embed_compute_rope_neox(cosv, sinv, positions, seq, head_dim, 10000.0f);
+ embed_apply_rope_neox(x, cosv, sinv, seq, heads, head_dim);
for (int s = 0; s < seq; s++) {
for (int d = 0; d < head_dim / 2; d++) {
@@ -225,7 +225,7 @@ static void test_packed_gqa_attention(void) {
V[i] = sinf((float)i * 0.07f) - cosf((float)i * 0.13f);
}
- qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
+ embed_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim));
reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim), 0);
@@ -254,12 +254,12 @@ static void test_packed_gqa_attention_long(void) {
V[i] = sinf((float)i * 0.007f) - cosf((float)i * 0.013f);
}
- qwen_set_threads(4);
- qwen_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
+ embed_set_threads(4);
+ embed_bidirectional_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim));
reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim), 0);
- qwen_set_threads(1);
+ embed_set_threads(1);
for (int i = 0; i < total * q_hidden; i++)
expect_close("packed_gqa_long", got[i], want[i], 2e-5f);
@@ -290,12 +290,12 @@ static void test_packed_causal_gqa_attention(void) {
V[i] = sinf((float)i * 0.009f) - cosf((float)i * 0.015f);
}
- qwen_set_threads(4);
- qwen_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
+ embed_set_threads(4);
+ embed_causal_gqa_attention_packed(got, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim));
reference_packed_gqa(want, Q, K, V, offsets, 2, n_heads, n_kv_heads, head_dim,
1.0f / sqrtf((float)head_dim), 1);
- qwen_set_threads(1);
+ embed_set_threads(1);
for (int i = 0; i < total * q_hidden; i++)
expect_close("packed_causal_gqa", got[i], want[i], 2e-5f);
@@ -315,7 +315,7 @@ static void test_bf16_linear(void) {
for (int i = 0; i < 6; i++)
wbf16[i] = f32_to_bf16(wf32[i]);
- qwen_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim);
+ embed_linear_nobias_bf16(got, x, wbf16, seq, in_dim, out_dim);
for (int s = 0; s < seq; s++) {
for (int o = 0; o < out_dim; o++) {
@@ -351,8 +351,8 @@ static void check_bf16_qkv(int n_threads) {
wv[i] = f32_to_bf16(wv_f32[i]);
}
- qwen_set_threads(n_threads);
- qwen_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim);
+ embed_set_threads(n_threads);
+ embed_linear_nobias_bf16_qkv(got_q, got_k, got_v, x, wq, wk, wv, seq, in_dim, q_dim, kv_dim);
for (int s = 0; s < seq; s++) {
for (int o = 0; o < q_dim; o++) {
@@ -376,7 +376,7 @@ static void check_bf16_qkv(int n_threads) {
static void test_bf16_qkv(void) {
check_bf16_qkv(1);
check_bf16_qkv(3);
- qwen_set_threads(1);
+ embed_set_threads(1);
}
/* seq==1 routes through bf16_matvec_threaded; a pool splits the output
@@ -392,8 +392,8 @@ static void check_bf16_matvec(int n_threads) {
for (int i = 0; i < out_dim * in_dim; i++)
w[i] = f32_to_bf16(cosf((float)i * 0.21f) + 0.05f);
- qwen_set_threads(n_threads);
- qwen_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim);
+ embed_set_threads(n_threads);
+ embed_linear_nobias_bf16(got, x, w, 1, in_dim, out_dim);
for (int o = 0; o < out_dim; o++) {
float want = 0.0f;
@@ -406,10 +406,10 @@ static void check_bf16_matvec(int n_threads) {
static void test_bf16_matvec(void) {
check_bf16_matvec(1);
check_bf16_matvec(4);
- qwen_set_threads(1);
+ embed_set_threads(1);
}
-/* Above QWEN_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to
+/* Above EMBED_RMS_NORM_PARALLEL_ELEMS (256k elements) rms_norm fans out to
* rms_norm_worker; row-wise math is identical, so results must match the
* single-thread run exactly. */
static void test_rms_norm_threaded(void) {
@@ -436,11 +436,11 @@ static void test_rms_norm_threaded(void) {
for (int i = 0; i < hidden; i++)
w[i] = 0.8f + 0.01f * (float)(i % 7);
- qwen_set_threads(1);
- qwen_rms_norm(one, x, w, seq, hidden, 1e-6f);
- qwen_set_threads(4);
- qwen_rms_norm(many, x, w, seq, hidden, 1e-6f);
- qwen_set_threads(1);
+ embed_set_threads(1);
+ embed_rms_norm(one, x, w, seq, hidden, 1e-6f);
+ embed_set_threads(4);
+ embed_rms_norm(many, x, w, seq, hidden, 1e-6f);
+ embed_set_threads(1);
for (size_t i = 0; i < elems; i++) {
if (one[i] != many[i]) {
@@ -469,7 +469,7 @@ static void test_bf16_widen_buf(void) {
for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) {
int n = sizes[t];
memset(got, 0, sizeof(got));
- qwen_bf16_to_f32_buf(got, src, (size_t)n);
+ embed_bf16_to_f32_buf(got, src, (size_t)n);
for (int i = 0; i < n; i++) {
float want = bf16_to_f32(src[i]);
uint32_t got_bits, want_bits;
@@ -503,8 +503,8 @@ static void check_bf16_pair(int n_threads) {
wb[i] = f32_to_bf16(wb_f32[i]);
}
- qwen_set_threads(n_threads);
- qwen_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim);
+ embed_set_threads(n_threads);
+ embed_linear_nobias_bf16_pair(got_a, got_b, x, wa, wb, seq, in_dim, a_dim, b_dim);
for (int s = 0; s < seq; s++) {
for (int o = 0; o < a_dim; o++) {
@@ -525,10 +525,10 @@ static void check_bf16_pair(int n_threads) {
static void test_bf16_pair(void) {
check_bf16_pair(1);
check_bf16_pair(4);
- qwen_set_threads(1);
+ embed_set_threads(1);
}
-/* The dispatch macros in qwen_kernels_impl.h pick the SIMD variant at
+/* The dispatch macros in embed_kernels_impl.h pick the SIMD variant at
* compile time, so on arm64/x86 the generic C kernels are linked but never
* called. Run them directly against the selected implementation on
* remainder-lane-hostile sizes; on a plain build impl == generic and the
@@ -554,46 +554,46 @@ static void test_generic_vs_impl(void) {
}
float yg[OUT], yi[OUT];
- qwen_bf16_matvec_fused_generic(yg, x, W, bias, IN, OUT);
- qwen_bf16_matvec_fused_impl(yi, x, W, bias, IN, OUT);
+ embed_bf16_matvec_fused_generic(yg, x, W, bias, IN, OUT);
+ embed_bf16_matvec_fused_impl(yi, x, W, bias, IN, OUT);
for (int o = 0; o < OUT; o++)
expect_close("matvec_fused generic-vs-impl", yg[o], yi[o], 1e-5f);
- qwen_bf16_matvec_fused_generic(yg, x, W, NULL, IN, OUT);
- qwen_bf16_matvec_fused_impl(yi, x, W, NULL, IN, OUT);
+ embed_bf16_matvec_fused_generic(yg, x, W, NULL, IN, OUT);
+ embed_bf16_matvec_fused_impl(yi, x, W, NULL, IN, OUT);
for (int o = 0; o < OUT; o++)
expect_close("matvec_fused nobias generic-vs-impl", yg[o], yi[o], 1e-5f);
const int sizes[] = {1, 7, 64, NMAX};
for (size_t t = 0; t < sizeof(sizes) / sizeof(sizes[0]); t++) {
int n = sizes[t];
- expect_close("dot generic-vs-impl", qwen_dot_f32_generic(a, b, n),
- qwen_dot_f32_impl(a, b, n), 1e-5f);
+ expect_close("dot generic-vs-impl", embed_dot_f32_generic(a, b, n),
+ embed_dot_f32_impl(a, b, n), 1e-5f);
memcpy(d0, a, (size_t)n * sizeof(float));
memcpy(d1, a, (size_t)n * sizeof(float));
- qwen_vec_scale_inplace_generic(d0, 1.37f, n);
- qwen_vec_scale_inplace_impl(d1, 1.37f, n);
+ embed_vec_scale_inplace_generic(d0, 1.37f, n);
+ embed_vec_scale_inplace_impl(d1, 1.37f, n);
for (int i = 0; i < n; i++)
expect_close("scale generic-vs-impl", d0[i], d1[i], 1e-6f);
memcpy(d0, a, (size_t)n * sizeof(float));
memcpy(d1, a, (size_t)n * sizeof(float));
- qwen_vec_axpy_inplace_generic(d0, b, -0.61f, n);
- qwen_vec_axpy_inplace_impl(d1, b, -0.61f, n);
+ embed_vec_axpy_inplace_generic(d0, b, -0.61f, n);
+ embed_vec_axpy_inplace_impl(d1, b, -0.61f, n);
for (int i = 0; i < n; i++)
expect_close("axpy generic-vs-impl", d0[i], d1[i], 1e-6f);
memcpy(d0, a, (size_t)n * sizeof(float));
memcpy(d1, a, (size_t)n * sizeof(float));
- qwen_vec_scale_add_generic(d0, b, 0.83f, n);
- qwen_vec_scale_add_impl(d1, b, 0.83f, n);
+ embed_vec_scale_add_generic(d0, b, 0.83f, n);
+ embed_vec_scale_add_impl(d1, b, 0.83f, n);
for (int i = 0; i < n; i++)
expect_close("scale_add generic-vs-impl", d0[i], d1[i], 1e-6f);
}
}
int main(void) {
- qwen_set_threads(1);
+ embed_set_threads(1);
test_rms_norm();
test_layer_norm();
test_gelu();
diff --git a/tests/test_late.c b/tests/test_late.c
index 661661c..ec6c03f 100644
--- a/tests/test_late.c
+++ b/tests/test_late.c
@@ -3,7 +3,7 @@
* byte-complete tokenizer fixture. Runs via `make test`. */
#include "embed.h"
-#include "qwen_tokenizer.h"
+#include "tokenizer_bpe.h"
#include "tiny_model.h"
#include "tok_fixture.h"
@@ -63,7 +63,7 @@ int main(void) {
tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE};
int hidden = dims.hidden;
- qwen_tokenizer_t *tok = NULL;
+ embed_tokenizer_t *tok = NULL;
embed_model_t *base = NULL;
embed_workspace_t *base_ws = NULL;
embed_late_model_t *late = NULL;
@@ -145,14 +145,14 @@ int main(void) {
char vocab_path[1280];
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir);
- tok = qwen_tokenizer_load(vocab_path);
+ tok = embed_tokenizer_load(vocab_path);
if (!tok) {
fprintf(stderr, "tokenizer load failed\n");
goto fail;
}
int n_tokens = 0;
- ids = qwen_tokenizer_encode(tok, "hello world", &n_tokens);
+ ids = embed_tokenizer_encode(tok, "hello world", &n_tokens);
if (!ids || n_tokens <= 1) {
fprintf(stderr, "tokenization failed\n");
goto fail;
@@ -244,7 +244,7 @@ int main(void) {
int lens[3] = {0};
int offsets[4] = {0};
for (int i = 0; i < 3; i++) {
- doc_ids[i] = qwen_tokenizer_encode(tok, doc_texts[i], &lens[i]);
+ doc_ids[i] = embed_tokenizer_encode(tok, doc_texts[i], &lens[i]);
if (!doc_ids[i] || lens[i] <= 0) {
fprintf(stderr, "doc tokenization failed\n");
goto fail;
@@ -401,7 +401,7 @@ fail:
free(raw);
free(states);
free(ids);
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_late_workspace_free(ws);
embed_late_model_free(late);
embed_workspace_free(base_ws);
diff --git a/tests/test_safetensors.c b/tests/test_safetensors.c
index 36ec8c5..ccfd6fb 100644
--- a/tests/test_safetensors.c
+++ b/tests/test_safetensors.c
@@ -2,7 +2,7 @@
* Hermetic: synthesizes every fixture in a temp directory. Runs via
* `make test`. */
-#include "qwen_safetensors.h"
+#include "safetensors.h"
#include <stdint.h>
#include <stdio.h>
diff --git a/tests/test_server.c b/tests/test_server.c
index 9ebb8f2..c0c85f1 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -6,7 +6,7 @@
* path over loopback sockets. Runs via `make test`. */
#define EMBED_SERVER_TEST
-#include "../embed_server.c"
+#include "../src/server.c"
#include "tiny_model.h"
#include "tok_fixture.h"
@@ -596,7 +596,7 @@ static void test_http_embeddings(int port) {
"{\"model\":\"Qwen3-Embedding-0.6B\","
"\"input\":\"hello\",\"encoding_format\":\"float\"}",
NULL, &body) == 200);
- cJSON *qwen_full = body ? parse_embeddings_body(body, 1) : NULL;
+ cJSON *embed_full = body ? parse_embeddings_body(body, 1) : NULL;
free(body);
body = NULL;
TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
@@ -604,10 +604,10 @@ static void test_http_embeddings(int port) {
"\"input\":\"hello\",\"dimensions\":32,"
"\"encoding_format\":\"float\"}",
NULL, &body) == 200);
- cJSON *qwen_mrl = body ? parse_embeddings_body(body, 1) : NULL;
- if (qwen_full && qwen_mrl) {
- cJSON *full_data = cJSON_GetObjectItemCaseSensitive(qwen_full, "data");
- cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "data");
+ cJSON *embed_mrl = body ? parse_embeddings_body(body, 1) : NULL;
+ if (embed_full && embed_mrl) {
+ cJSON *full_data = cJSON_GetObjectItemCaseSensitive(embed_full, "data");
+ cJSON *mrl_data = cJSON_GetObjectItemCaseSensitive(embed_mrl, "data");
cJSON *full_emb =
cJSON_GetObjectItemCaseSensitive(cJSON_GetArrayItem(full_data, 0), "embedding");
cJSON *mrl_emb =
@@ -622,12 +622,12 @@ static void test_http_embeddings(int port) {
prefix_diff = d;
}
TEST_ASSERT(prefix_diff > 0.0);
- cJSON *usage = cJSON_GetObjectItemCaseSensitive(qwen_mrl, "usage");
+ cJSON *usage = cJSON_GetObjectItemCaseSensitive(embed_mrl, "usage");
cJSON *total = usage ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
TEST_ASSERT(total && total->valueint == 2);
}
- cJSON_Delete(qwen_full);
- cJSON_Delete(qwen_mrl);
+ cJSON_Delete(embed_full);
+ cJSON_Delete(embed_mrl);
free(body);
body = NULL;
@@ -949,7 +949,7 @@ static void test_http_rerank(int port) {
}
static void test_http_server(void) {
- char dir[1024], late_dir[1024], qwen_dir[1024], gte_dir[1024];
+ char dir[1024], late_dir[1024], embed_dir[1024], gte_dir[1024];
snprintf(dir, sizeof(dir), "%s/embed-srv-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
if (!mkdtemp(dir)) {
@@ -964,9 +964,9 @@ static void test_http_server(void) {
test_failures++;
return;
}
- snprintf(qwen_dir, sizeof(qwen_dir), "%s/embed-srv-qwen-test-XXXXXX",
+ snprintf(embed_dir, sizeof(embed_dir), "%s/embed-srv-qwen-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
- if (!mkdtemp(qwen_dir)) {
+ if (!mkdtemp(embed_dir)) {
fprintf(stderr, "FAIL: Qwen mkdtemp\n");
test_failures++;
return;
@@ -985,8 +985,8 @@ static void test_http_server(void) {
if (tf_write_vocab(dir) != 0 || tm_write_model_dims(dir, "F32", &dims) != 0 ||
tf_write_vocab(late_dir) != 0 || tm_write_model_dims(late_dir, "F32", &dims) != 0 ||
tm_write_late_projection(late_dir, "F32", 128, 1024) != 0 ||
- tf_write_vocab(qwen_dir) != 0 ||
- tm_write_qwen3_model_dims(qwen_dir, "F32", &dims, TF_EOT_ID) != 0 ||
+ tf_write_vocab(embed_dir) != 0 ||
+ tm_write_qwen3_model_dims(embed_dir, "F32", &dims, TF_EOT_ID) != 0 ||
tf_write_vocab(gte_dir) != 0 ||
tm_write_qwen2_model_dims(gte_dir, "F32", &gte_dims, TF_EOT_ID, 0) != 0) {
fprintf(stderr, "FAIL: fixture write\n");
@@ -1008,7 +1008,7 @@ static void test_http_server(void) {
ctx.spec[2].id = "pplx-embed-v1-late-0.6b";
ctx.spec[2].path = late_dir;
ctx.spec[3].id = "Qwen3-Embedding-0.6B";
- ctx.spec[3].path = qwen_dir;
+ ctx.spec[3].path = embed_dir;
ctx.spec[4].id = "gte-Qwen2-1.5B-instruct";
ctx.spec[4].path = gte_dir;
ctx.cfg.models = ctx.spec;
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c
index 5920977..3ef4280 100644
--- a/tests/test_tokenizer.c
+++ b/tests/test_tokenizer.c
@@ -4,7 +4,7 @@
* integer-pair fast path, and that dropping the integer tables (forcing the
* string-based fallback) produces identical ids. Runs via `make test`. */
-#include "qwen_tokenizer.h"
+#include "tokenizer_bpe.h"
#include "tok_fixture.h"
#include <stdio.h>
@@ -14,7 +14,7 @@
#include <unistd.h>
static int failures = 0;
-int qwen_verbose = 0;
+int embed_verbose = 0;
#define TEST_ASSERT(cond) \
do { \
@@ -24,14 +24,14 @@ int qwen_verbose = 0;
} \
} while (0)
-static void check_ids(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
+static void check_ids(const embed_tokenizer_t *tok,
+ embed_tokenizer_workspace_t *ws,
const char *text,
const int *want,
int n_want,
const char *label) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n);
+ int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n);
TEST_ASSERT(ids != NULL);
if (!ids)
return;
@@ -52,19 +52,19 @@ static void check_ids(const qwen_tokenizer_t *tok,
/* Concatenated decode of an encoding must reproduce the (NFC-normalized)
* input - byte-level BPE is lossless. */
-static void check_roundtrip(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
+static void check_roundtrip(const embed_tokenizer_t *tok,
+ embed_tokenizer_workspace_t *ws,
const char *text,
const char *want,
const char *label) {
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, text, &n);
+ int *ids = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, text, &n);
TEST_ASSERT(ids != NULL && n > 0);
if (!ids)
return;
size_t cap = 1;
for (int i = 0; i < n; i++)
- cap += strlen(qwen_tokenizer_decode(tok, ids[i]));
+ cap += strlen(embed_tokenizer_decode(tok, ids[i]));
char *got = (char *)malloc(cap);
if (!got) {
free(ids);
@@ -73,7 +73,7 @@ static void check_roundtrip(const qwen_tokenizer_t *tok,
}
got[0] = '\0';
for (int i = 0; i < n; i++)
- strcat(got, qwen_tokenizer_decode(tok, ids[i]));
+ strcat(got, embed_tokenizer_decode(tok, ids[i]));
if (strcmp(got, want) != 0) {
fprintf(stderr, "FAIL %s roundtrip \"%s\": got \"%s\" want \"%s\"\n", label, text, got,
want);
@@ -84,18 +84,18 @@ static void check_roundtrip(const qwen_tokenizer_t *tok,
}
/* Two spellings of the same text must encode identically. */
-static void check_same_ids(const qwen_tokenizer_t *tok,
- qwen_tokenizer_workspace_t *ws,
+static void check_same_ids(const embed_tokenizer_t *tok,
+ embed_tokenizer_workspace_t *ws,
const char *a,
const char *b,
const char *label) {
int na = 0, nb = 0;
- int *ia = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, a, &na);
+ int *ia = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, a, &na);
if (!ia) {
TEST_ASSERT(0);
return;
}
- int *ib = qwen_tokenizer_encode_with_workspace((qwen_tokenizer_t *)tok, ws, b, &nb);
+ int *ib = embed_tokenizer_encode_with_workspace((embed_tokenizer_t *)tok, ws, b, &nb);
if (!ib) {
free(ia);
TEST_ASSERT(0);
@@ -110,7 +110,7 @@ static void check_same_ids(const qwen_tokenizer_t *tok,
}
static void
-run_cases(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *label) {
+run_cases(const embed_tokenizer_t *tok, embed_tokenizer_workspace_t *ws, const char *label) {
/* h e l l o -> he -> he,ll -> hell -> hello */
const int hello[] = {259};
check_ids(tok, ws, "hello", hello, 1, label);
@@ -212,7 +212,7 @@ static void write_file(const char *dir, const char *name, const char *body) {
static void test_loader_edges(const char *root) {
char dir[1152], path[2048];
- TEST_ASSERT(qwen_tokenizer_load("/nonexistent/vocab.json") == NULL);
+ TEST_ASSERT(embed_tokenizer_load("/nonexistent/vocab.json") == NULL);
/* Not a JSON object. */
snprintf(dir, sizeof(dir), "%s/badjson", root);
@@ -223,7 +223,7 @@ static void test_loader_edges(const char *root) {
write_file(dir, "vocab.json", "[1, 2]\n");
write_file(dir, "merges.txt", "a b\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
- TEST_ASSERT(qwen_tokenizer_load(path) == NULL);
+ TEST_ASSERT(embed_tokenizer_load(path) == NULL);
/* Missing or pair-less merges.txt is tolerated: the tokenizer loads
* and encoding falls back to byte/vocab-level (no merges applied). */
@@ -234,16 +234,16 @@ static void test_loader_edges(const char *root) {
}
write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
- qwen_tokenizer_t *nom = qwen_tokenizer_load(path);
+ embed_tokenizer_t *nom = embed_tokenizer_load(path);
TEST_ASSERT(nom != NULL);
if (nom) {
- qwen_tokenizer_workspace_t *nws = qwen_tokenizer_workspace_new();
+ embed_tokenizer_workspace_t *nws = embed_tokenizer_workspace_new();
int n = 0;
- int *ids = qwen_tokenizer_encode_with_workspace(nom, nws, "ab", &n);
+ int *ids = embed_tokenizer_encode_with_workspace(nom, nws, "ab", &n);
TEST_ASSERT(ids && n == 2 && ids[0] == 0 && ids[1] == 1);
free(ids);
- qwen_tokenizer_workspace_free(nws);
- qwen_tokenizer_free(nom);
+ embed_tokenizer_workspace_free(nws);
+ embed_tokenizer_free(nom);
}
snprintf(dir, sizeof(dir), "%s/emptymerges", root);
@@ -254,9 +254,9 @@ static void test_loader_edges(const char *root) {
write_file(dir, "vocab.json", "{\"a\":0,\"b\":1}\n");
write_file(dir, "merges.txt", "#version: 0.2\n\n \n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
- qwen_tokenizer_t *emp = qwen_tokenizer_load(path);
+ embed_tokenizer_t *emp = embed_tokenizer_load(path);
TEST_ASSERT(emp != NULL);
- qwen_tokenizer_free(emp);
+ embed_tokenizer_free(emp);
/* Escape forms in vocab keys; a malformed merge line is skipped. */
snprintf(dir, sizeof(dir), "%s/escapes", root);
@@ -270,28 +270,28 @@ static void test_loader_edges(const char *root) {
" \"s\\\\t\":6, \"p\\/q\":7, \"u\\tv\":8, \"\\u0041Z\":9}\n");
write_file(dir, "merges.txt", "#version: 0.2\nmalformed-line\na b\n");
snprintf(path, sizeof(path), "%s/vocab.json", dir);
- qwen_tokenizer_t *tok = qwen_tokenizer_load(path);
+ embed_tokenizer_t *tok = embed_tokenizer_load(path);
TEST_ASSERT(tok != NULL);
if (tok) {
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "x\ny") == 4);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "q\"r") == 5);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "s\\t") == 6);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "p/q") == 7);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "u\tv") == 8);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "AZ") == 9);
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 2), "ab") == 0);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "\xE2\x82\xAC") == 3);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "x\ny") == 4);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "q\"r") == 5);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "s\\t") == 6);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "p/q") == 7);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "u\tv") == 8);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "AZ") == 9);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 2), "ab") == 0);
/* Keys outside the GPT-2 byte alphabet decode to replacement. */
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 3), "?") == 0);
- qwen_tokenizer_free(tok);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 3), "?") == 0);
+ embed_tokenizer_free(tok);
}
/* A bare filename exercises the no-directory merges-path branch. */
char saved_cwd[1024];
if (getcwd(saved_cwd, sizeof(saved_cwd)) && chdir(dir) == 0) {
- qwen_tokenizer_t *rel = qwen_tokenizer_load("vocab.json");
+ embed_tokenizer_t *rel = embed_tokenizer_load("vocab.json");
TEST_ASSERT(rel != NULL);
- qwen_tokenizer_free(rel);
+ embed_tokenizer_free(rel);
if (chdir(saved_cwd) != 0) {
perror("chdir back");
exit(2);
@@ -310,7 +310,7 @@ int main(void) {
char vocab_path[2048];
snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir);
- qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path);
TEST_ASSERT(tok != NULL);
if (!tok)
return 1;
@@ -319,48 +319,48 @@ int main(void) {
TEST_ASSERT(tok->int_merges != NULL);
TEST_ASSERT(tok->cp_to_id != NULL);
- qwen_tokenizer_workspace_t *ws = qwen_tokenizer_workspace_new();
+ embed_tokenizer_workspace_t *ws = embed_tokenizer_workspace_new();
TEST_ASSERT(ws != NULL);
run_cases(tok, ws, "fast");
/* decode: byte tokens, merged tokens, and out-of-range ids. */
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 'a'), "a") == 0);
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, 259), "hello") == 0);
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, -1), "") == 0);
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0);
- TEST_ASSERT(strcmp(qwen_tokenizer_decode(NULL, 0), "") == 0);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 'a'), "a") == 0);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, 259), "hello") == 0);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, -1), "") == 0);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(tok, TF_VOCAB_SIZE), "") == 0);
+ TEST_ASSERT(strcmp(embed_tokenizer_decode(NULL, 0), "") == 0);
/* token_id lookups. */
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "hello") == 259);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, "no-such-token") == -1);
- TEST_ASSERT(qwen_tokenizer_token_id(NULL, "a") == -1);
- TEST_ASSERT(qwen_tokenizer_token_id(tok, NULL) == -1);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "hello") == 259);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "<|endoftext|>") == TF_EOT_ID);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, "no-such-token") == -1);
+ TEST_ASSERT(embed_tokenizer_token_id(NULL, "a") == -1);
+ TEST_ASSERT(embed_tokenizer_token_id(tok, NULL) == -1);
/* encode_into: exact fit, truncation (-2 plus the untruncated prefix),
* zero capacity, empty text, and argument guards. */
{
int out[8];
int n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3);
+ TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 8, &n) == 0 && n == 3);
TEST_ASSERT(out[0] == 259 && out[1] == 263 && out[2] == 262);
/* On overflow the prefix is written and n reports the total
* required count, so a caller can re-size and retry. */
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3);
+ TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 2, &n) == -2 && n == 3);
TEST_ASSERT(out[0] == 259 && out[1] == 263);
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3);
+ TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "hello world", out, 0, &n) == -2 && n == 3);
n = -1;
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0);
- TEST_ASSERT(qwen_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1);
- TEST_ASSERT(qwen_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1);
+ TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, "", out, 8, &n) == 0 && n == 0);
+ TEST_ASSERT(embed_tokenizer_encode_into(tok, ws, NULL, out, 8, &n) == -1);
+ TEST_ASSERT(embed_tokenizer_encode_into(NULL, ws, "x", out, 8, &n) == -1);
}
- TEST_ASSERT(qwen_tokenizer_encode(tok, "", NULL) == NULL);
- TEST_ASSERT(qwen_tokenizer_encode(NULL, "x", NULL) == NULL);
- qwen_tokenizer_workspace_free(NULL);
- qwen_tokenizer_free(NULL);
+ TEST_ASSERT(embed_tokenizer_encode(tok, "", NULL) == NULL);
+ TEST_ASSERT(embed_tokenizer_encode(NULL, "x", NULL) == NULL);
+ embed_tokenizer_workspace_free(NULL);
+ embed_tokenizer_free(NULL);
/* Drop the integer tables: the string-based fallback must produce the
* exact same ids. */
@@ -371,8 +371,8 @@ int main(void) {
tok->cp_to_id = NULL;
run_cases(tok, ws, "slow");
- qwen_tokenizer_workspace_free(ws);
- qwen_tokenizer_free(tok);
+ embed_tokenizer_workspace_free(ws);
+ embed_tokenizer_free(tok);
test_loader_edges(dir);
diff --git a/tests/test_wordpiece.c b/tests/test_wordpiece.c
index 1fd6ff8..d40b7d2 100644
--- a/tests/test_wordpiece.c
+++ b/tests/test_wordpiece.c
@@ -3,7 +3,7 @@
* punctuation splitting, CJK char-splitting, special-token resolution, and the
* workspace encode paths. No model files or network needed. */
-#include "wordpiece_tokenizer.h"
+#include "tokenizer_wordpiece.h"
#include <stdio.h>
#include <stdlib.h>
diff --git a/tests/test_workspace.c b/tests/test_workspace.c
index a906689..c8b6ee8 100644
--- a/tests/test_workspace.c
+++ b/tests/test_workspace.c
@@ -4,7 +4,7 @@
* what scripts/check_workspace_api.py does. Runs via `make test`. */
#include "embed.h"
-#include "qwen_tokenizer.h"
+#include "tokenizer_bpe.h"
#include "tiny_model.h"
#include "tok_fixture.h"
@@ -199,10 +199,10 @@ int main(int argc, char **argv) {
embed_workspace_t *batch_ws = embed_workspace_new(model);
embed_workspace_t *single_ws = embed_workspace_new(model);
- qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path);
if (!batch_ws || !single_ws || !tok) {
fprintf(stderr, "failed to allocate workspace or tokenizer\n");
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);
@@ -227,7 +227,7 @@ int main(int argc, char **argv) {
}
for (int i = 0; i < batch; i++) {
- ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]);
+ ids[i] = embed_tokenizer_encode(tok, texts[i], &ntok[i]);
if (!ids[i] || ntok[i] <= 0) {
fprintf(stderr, "failed to tokenize input %d\n", i);
goto fail;
@@ -331,7 +331,7 @@ int main(int argc, char **argv) {
/* Same separator resolution as the server: the fixture vocab defines
* <|endoftext|>, real snapshots use the reserved id. */
- int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>");
+ int sep_id = embed_tokenizer_token_id(tok, "<|endoftext|>");
if (sep_id < 0)
sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) {
@@ -349,7 +349,7 @@ int main(int argc, char **argv) {
for (int i = 0; i < batch; i++)
free(ids[i]);
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);
@@ -367,7 +367,7 @@ int main(int argc, char **argv) {
fail:
for (int i = 0; i < batch; i++)
free(ids[i]);
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);