diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /tests/test_workspace.c | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'tests/test_workspace.c')
| -rw-r--r-- | tests/test_workspace.c | 14 |
1 files changed, 7 insertions, 7 deletions
diff --git a/tests/test_workspace.c b/tests/test_workspace.c index a906689..c8b6ee8 100644 --- a/tests/test_workspace.c +++ b/tests/test_workspace.c @@ -4,7 +4,7 @@ * what scripts/check_workspace_api.py does. Runs via `make test`. */ #include "embed.h" -#include "qwen_tokenizer.h" +#include "tokenizer_bpe.h" #include "tiny_model.h" #include "tok_fixture.h" @@ -199,10 +199,10 @@ int main(int argc, char **argv) { embed_workspace_t *batch_ws = embed_workspace_new(model); embed_workspace_t *single_ws = embed_workspace_new(model); - qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path); + embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path); if (!batch_ws || !single_ws || !tok) { fprintf(stderr, "failed to allocate workspace or tokenizer\n"); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); @@ -227,7 +227,7 @@ int main(int argc, char **argv) { } for (int i = 0; i < batch; i++) { - ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]); + ids[i] = embed_tokenizer_encode(tok, texts[i], &ntok[i]); if (!ids[i] || ntok[i] <= 0) { fprintf(stderr, "failed to tokenize input %d\n", i); goto fail; @@ -331,7 +331,7 @@ int main(int argc, char **argv) { /* Same separator resolution as the server: the fixture vocab defines * <|endoftext|>, real snapshots use the reserved id. */ - int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>"); + int sep_id = embed_tokenizer_token_id(tok, "<|endoftext|>"); if (sep_id < 0) sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID; if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) { @@ -349,7 +349,7 @@ int main(int argc, char **argv) { for (int i = 0; i < batch; i++) free(ids[i]); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); @@ -367,7 +367,7 @@ int main(int argc, char **argv) { fail: for (int i = 0; i < batch; i++) free(ids[i]); - qwen_tokenizer_free(tok); + embed_tokenizer_free(tok); embed_workspace_free(single_ws); embed_workspace_free(batch_ws); embed_model_free(model); |
