aboutsummaryrefslogtreecommitdiff
path: root/tests/test_workspace.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-16 16:27:47 +0200
commitd1ba1ebc06ef0d62ca932388fae83e6992594719 (patch)
treef962207b779016593258f990348216999a58766a /tests/test_workspace.c
parentbe20dfda0f836808dccfc14db244592e41ba28bc (diff)
downloadffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz
ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'tests/test_workspace.c')
-rw-r--r--tests/test_workspace.c14
1 files changed, 7 insertions, 7 deletions
diff --git a/tests/test_workspace.c b/tests/test_workspace.c
index a906689..c8b6ee8 100644
--- a/tests/test_workspace.c
+++ b/tests/test_workspace.c
@@ -4,7 +4,7 @@
* what scripts/check_workspace_api.py does. Runs via `make test`. */
#include "embed.h"
-#include "qwen_tokenizer.h"
+#include "tokenizer_bpe.h"
#include "tiny_model.h"
#include "tok_fixture.h"
@@ -199,10 +199,10 @@ int main(int argc, char **argv) {
embed_workspace_t *batch_ws = embed_workspace_new(model);
embed_workspace_t *single_ws = embed_workspace_new(model);
- qwen_tokenizer_t *tok = qwen_tokenizer_load(vocab_path);
+ embed_tokenizer_t *tok = embed_tokenizer_load(vocab_path);
if (!batch_ws || !single_ws || !tok) {
fprintf(stderr, "failed to allocate workspace or tokenizer\n");
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);
@@ -227,7 +227,7 @@ int main(int argc, char **argv) {
}
for (int i = 0; i < batch; i++) {
- ids[i] = qwen_tokenizer_encode(tok, texts[i], &ntok[i]);
+ ids[i] = embed_tokenizer_encode(tok, texts[i], &ntok[i]);
if (!ids[i] || ntok[i] <= 0) {
fprintf(stderr, "failed to tokenize input %d\n", i);
goto fail;
@@ -331,7 +331,7 @@ int main(int argc, char **argv) {
/* Same separator resolution as the server: the fixture vocab defines
* <|endoftext|>, real snapshots use the reserved id. */
- int sep_id = qwen_tokenizer_token_id(tok, "<|endoftext|>");
+ int sep_id = embed_tokenizer_token_id(tok, "<|endoftext|>");
if (sep_id < 0)
sep_id = EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
if (check_spans_batch_parity(model, batch_ws, ids, ntok, sep_id, cfg) != 0) {
@@ -349,7 +349,7 @@ int main(int argc, char **argv) {
for (int i = 0; i < batch; i++)
free(ids[i]);
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);
@@ -367,7 +367,7 @@ int main(int argc, char **argv) {
fail:
for (int i = 0; i < batch; i++)
free(ids[i]);
- qwen_tokenizer_free(tok);
+ embed_tokenizer_free(tok);
embed_workspace_free(single_ws);
embed_workspace_free(batch_ws);
embed_model_free(model);