aboutsummaryrefslogtreecommitdiff
path: root/tests/test_sentencepiece.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-17 01:21:41 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-17 01:21:41 +0200
commite00becb643b3a99de16462b3a032717a93b983d9 (patch)
tree873ad2646b0ef2f07eae5a5a74387d062c77ec0c /tests/test_sentencepiece.c
parentb4343d6ac1c6cc7a7d228f954220def2f6e6ed67 (diff)
downloadffwd-e00becb643b3a99de16462b3a032717a93b983d9.tar.gz
ffwd-e00becb643b3a99de16462b3a032717a93b983d9.zip
all: accept F16 safetensors weights (widen to F32 at load)
Diffstat (limited to 'tests/test_sentencepiece.c')
-rw-r--r--tests/test_sentencepiece.c39
1 files changed, 30 insertions, 9 deletions
diff --git a/tests/test_sentencepiece.c b/tests/test_sentencepiece.c
index b597af2..38bcb36 100644
--- a/tests/test_sentencepiece.c
+++ b/tests/test_sentencepiece.c
@@ -167,14 +167,22 @@ static int run_hermetic(void) {
return 2;
}
+ /* A directory with no SentencePiece model must fail cleanly, not crash. */
+ char empty[1024];
+ snprintf(empty, sizeof(empty), "%s/embed-sp-empty-XXXXXX", tmp);
+ if (mkdtemp(empty) && sentencepiece_tokenizer_load(empty) != NULL) {
+ fprintf(stderr, "load of model-less dir should fail\n");
+ return 1;
+ }
+
sentencepiece_tokenizer_t *tok = sentencepiece_tokenizer_load(dir);
sentencepiece_workspace_t *ws = sentencepiece_workspace_new();
if (!tok || !ws) {
fprintf(stderr, "load failed\n");
return 1;
}
- if (tok->piece_count != 12 || tok->vocab_size != 14 || tok->unk_id != 3 ||
- tok->bos_id != 0 || tok->eos_id != 2 || tok->pad_id != 1 || tok->mask_id != 13) {
+ if (tok->piece_count != 12 || tok->vocab_size != 14 || tok->unk_id != 3 || tok->bos_id != 0 ||
+ tok->eos_id != 2 || tok->pad_id != 1 || tok->mask_id != 13) {
fprintf(stderr, "metadata wrong\n");
return 1;
}
@@ -201,6 +209,17 @@ static int run_hermetic(void) {
fprintf(stderr, "small-buffer handling wrong\n");
return 1;
}
+
+ /* The no-workspace encode entry point must agree with the workspace path. */
+ int plain_n = -1;
+ int *plain = sentencepiece_tokenizer_encode(tok, "Hello world!", &plain_n);
+ if (plain_n != 3 || !plain || plain[0] != 8 || plain[1] != 9 || plain[2] != 7) {
+ fprintf(stderr, "plain encode mismatch: n=%d\n", plain_n);
+ free(plain);
+ return 1;
+ }
+ free(plain);
+
sentencepiece_workspace_free(ws);
sentencepiece_tokenizer_free(tok);
@@ -232,17 +251,19 @@ static int run_live(const char *model_dir) {
int rc = 0;
rc |= check_ids(tok, ws, "Hello world! \xe5\x8d\x97\xe7\x93\x9c",
(int[]){35378, 8999, 38, 6, 4617, 39613}, 6, "live-mixed");
- rc |= check_ids(tok, ws, "caf\xc3\xa9 d\xc3\xa9j\xc3\xa0 vu",
- (int[]){26216, 15154, 13946}, 3, "live-accents");
- rc |= check_ids(tok, ws, "\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3 \xef\xbc\x91\xef\xbc\x92\xef\xbc\x93",
+ rc |= check_ids(tok, ws, "caf\xc3\xa9 d\xc3\xa9j\xc3\xa0 vu", (int[]){26216, 15154, 13946}, 3,
+ "live-accents");
+ rc |= check_ids(tok, ws,
+ "\xef\xbc\xa1\xef\xbc\xa2\xef\xbc\xa3 \xef\xbc\x91\xef\xbc\x92\xef\xbc\x93",
(int[]){47457, 37638}, 2, "live-nfkc");
rc |= check_ids(tok, ws, "query: what is snowflake?",
- (int[]){41, 1294, 12, 2367, 83, 108203, 13034, 350, 32}, 9,
- "live-query");
- rc |= check_ids(tok, ws, "\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbc\xd0\xb8\xd1\x80",
+ (int[]){41, 1294, 12, 2367, 83, 108203, 13034, 350, 32}, 9, "live-query");
+ rc |= check_ids(tok, ws,
+ "\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82 \xd0\xbc\xd0\xb8\xd1\x80",
(int[]){1813, 18454, 11373}, 3, "live-cyrillic");
rc |= check_ids(tok, ws,
- "\xd9\x85\xd8\xb1\xd8\xad\xd8\xa8\xd8\xa7 \xd8\xa8\xd8\xa7\xd9\x84\xd8\xb9\xd8\xa7\xd9\x84\xd9\x85",
+ "\xd9\x85\xd8\xb1\xd8\xad\xd8\xa8\xd8\xa7 "
+ "\xd8\xa8\xd8\xa7\xd9\x84\xd8\xb9\xd8\xa7\xd9\x84\xd9\x85",
(int[]){665, 193478, 258, 1705, 77796}, 5, "live-arabic");
rc |= check_ids(tok, ws, "", NULL, 0, "live-empty");
rc |= check_ids(tok, ws, " ", NULL, 0, "live-spaces");