aboutsummaryrefslogtreecommitdiff
path: root/qwen_tokenizer.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 16:12:18 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit0d422762c97071f446f7461c73adab55565db582 (patch)
treefce09da06fc8593aabc3b7f55510867826a7b2e9 /qwen_tokenizer.c
parent29c5ca74e9882d1ac8644aaa9c546cad66957b5c (diff)
downloadffwd-0d422762c97071f446f7461c73adab55565db582.tar.gz
ffwd-0d422762c97071f446f7461c73adab55565db582.zip
tests: tokenizer decode/NFC/contraction/loader-edge coverage
Diffstat (limited to 'qwen_tokenizer.c')
-rw-r--r--qwen_tokenizer.c15
1 files changed, 2 insertions, 13 deletions
diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c
index ee95326..59f8784 100644
--- a/qwen_tokenizer.c
+++ b/qwen_tokenizer.c
@@ -55,17 +55,6 @@ static int utf8_encode_cp(int cp, char out[4]) {
return 3;
}
-static int utf8_encode_full_cp(int cp, char out[4]) {
- if (cp < 0x80) return utf8_encode_cp(cp, out);
- if (cp < 0x800) return utf8_encode_cp(cp, out);
- if (cp < 0x10000) return utf8_encode_cp(cp, out);
- out[0] = (char)(0xF0 | (cp >> 18));
- out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
- out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
- out[3] = (char)(0x80 | (cp & 0x3F));
- return 4;
-}
-
/*
* Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text.
* Returns allocated string (caller must free).
@@ -635,7 +624,7 @@ static char *normalize_nfc_latin(const char *text) {
int composed = compose_latin_mark(cp, mark);
if (composed) {
char tmp[4];
- int n = utf8_encode_full_cp(composed, tmp);
+ int n = utf8_encode_cp(composed, tmp);
for (int i = 0; i < n; i++) out[w++] = tmp[i];
p += mark_len;
continue;
@@ -1039,7 +1028,7 @@ static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws,
int composed = compose_latin_mark(cp, mark);
if (composed) {
char tmp[4];
- int n = utf8_encode_full_cp(composed, tmp);
+ int n = utf8_encode_cp(composed, tmp);
memcpy(ws->normalized + w, tmp, (size_t)n);
w += (size_t)n;
p += mark_len;