diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /tests/test_late.c | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'tests/test_late.c')
| -rw-r--r-- | tests/test_late.c | 12 |
1 files changed, 6 insertions, 6 deletions
diff --git a/tests/test_late.c b/tests/test_late.c index 661661c..ec6c03f 100644 --- a/tests/test_late.c +++ b/tests/test_late.c @@ -3,7 +3,7 @@ * byte-complete tokenizer fixture. Runs via `make test`. */ #include "embed.h" -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tiny_model.h" #include "tok_fixture.h" @@ -63,7 +63,7 @@ int main(void) { tm_dims_t dims = {4, 2, 1, 2, 8, TF_VOCAB_SIZE}; int hidden = dims.hidden; - qwen_tokenizer_t *tok = NULL; + embed_tokenizer_t *tok = NULL; embed_model_t *base = NULL; embed_workspace_t *base_ws = NULL; embed_late_model_t *late = NULL; @@ -145,14 +145,14 @@ int main(void) { char vocab_path[1280]; snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", dir); - tok = qwen_tokenizer_load(vocab_path); + tok = embed_tokenizer_load(vocab_path); if (!tok) { fprintf(stderr, "tokenizer load failed\n"); goto fail; } int n_tokens = 0; - ids = qwen_tokenizer_encode(tok, "hello world", &n_tokens); + ids = embed_tokenizer_encode(tok, "hello world", &n_tokens); if (!ids || n_tokens <= 1) { fprintf(stderr, "tokenization failed\n"); goto fail; @@ -244,7 +244,7 @@ int main(void) { int lens[3] = {0}; int offsets[4] = {0}; for (int i = 0; i < 3; i++) { - doc_ids[i] = qwen_tokenizer_encode(tok, doc_texts[i], &lens[i]); + doc_ids[i] = embed_tokenizer_encode(tok, doc_texts[i], &lens[i]); if (!doc_ids[i] || lens[i] <= 0) { fprintf(stderr, "doc tokenization failed\n"); goto fail; @@ -401,7 +401,7 @@ fail: free(raw); free(states); free(ids); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_late_workspace_free(ws); embed_late_model_free(late); embed_workspace_free(base_ws); |
