diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-15 08:40:11 +0200 |
| commit | c4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch) | |
| tree | 81ce5675ce886aad0309a41f4d6a696826e0a061 /qwen_tokenizer.c | |
| parent | b377ba8656bfb807ed8dda6c9f26942b50ce626d (diff) | |
| download | ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip | |
chore: run clang-format
Diffstat (limited to 'qwen_tokenizer.c')
| -rw-r--r-- | qwen_tokenizer.c | 998 |
1 files changed, 630 insertions, 368 deletions
diff --git a/qwen_tokenizer.c b/qwen_tokenizer.c index 59f8784..e813071 100644 --- a/qwen_tokenizer.c +++ b/qwen_tokenizer.c @@ -15,24 +15,31 @@ static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ static int gpt2_mapping_initialized = 0; static void init_gpt2_mapping(void) { - if (gpt2_mapping_initialized) return; + if (gpt2_mapping_initialized) + return; memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte)); int n = 0; for (int b = 0; b < 256; b++) { int is_normal = 0; - if (b >= 33 && b <= 126) is_normal = 1; /* '!'..'~' */ - if (b >= 161 && b <= 172) is_normal = 1; /* '¡'..'¬' */ - if (b >= 174 && b <= 255) is_normal = 1; /* '®'..'ÿ' */ + if (b >= 33 && b <= 126) + is_normal = 1; /* '!'..'~' */ + if (b >= 161 && b <= 172) + is_normal = 1; /* '¡'..'¬' */ + if (b >= 174 && b <= 255) + is_normal = 1; /* '®'..'ÿ' */ - if (is_normal) gpt2_byte_to_unicode[b] = b; - else gpt2_byte_to_unicode[b] = 256 + n++; + if (is_normal) + gpt2_byte_to_unicode[b] = b; + else + gpt2_byte_to_unicode[b] = 256 + n++; } for (int b = 0; b < 256; b++) { int cp = gpt2_byte_to_unicode[b]; - if (cp < 512) gpt2_unicode_to_byte[cp] = b; + if (cp < 512) + gpt2_unicode_to_byte[cp] = b; } gpt2_mapping_initialized = 1; @@ -64,7 +71,8 @@ static char *decode_gpt2_token(const char *token_str) { size_t len = strlen(token_str); unsigned char *bytes = (unsigned char *)malloc(len + 1); - if (!bytes) return NULL; + if (!bytes) + return NULL; int byte_count = 0; const unsigned char *p = (const unsigned char *)token_str; @@ -79,18 +87,24 @@ static char *decode_gpt2_token(const char *token_str) { nbytes = 1; } else if ((*p & 0xE0) == 0xC0) { cp = (*p & 0x1F) << 6; - if (p + 1 < end) cp |= (p[1] & 0x3F); + if (p + 1 < end) + cp |= (p[1] & 0x3F); nbytes = 2; } else if ((*p & 0xF0) == 0xE0) { cp = (*p & 0x0F) << 12; - if (p + 1 < end) cp |= (p[1] & 0x3F) << 6; - if (p + 2 < end) cp |= (p[2] & 0x3F); + if (p + 1 < end) + cp |= (p[1] & 0x3F) << 6; + if (p + 2 < end) + cp |= (p[2] & 0x3F); nbytes = 3; } else if ((*p & 0xF8) == 0xF0) { cp = (*p & 0x07) << 18; - if (p + 1 < end) cp |= (p[1] & 0x3F) << 12; - if (p + 2 < end) cp |= (p[2] & 0x3F) << 6; - if (p + 3 < end) cp |= (p[3] & 0x3F); + if (p + 1 < end) + cp |= (p[1] & 0x3F) << 12; + if (p + 2 < end) + cp |= (p[2] & 0x3F) << 6; + if (p + 3 < end) + cp |= (p[3] & 0x3F); nbytes = 4; } else { cp = *p; @@ -113,31 +127,41 @@ static char *decode_gpt2_token(const char *token_str) { * ======================================================================== */ static void skip_ws(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') (*p)++; + while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') + (*p)++; } static int parse_json_string(const char **p, char *out, size_t max_len) { skip_ws(p); - if (**p != '"') return -1; + if (**p != '"') + return -1; (*p)++; size_t i = 0; while (**p && **p != '"' && i < max_len - 1) { if (**p == '\\') { (*p)++; - if (**p == 'n') out[i++] = '\n'; - else if (**p == 't') out[i++] = '\t'; - else if (**p == '"') out[i++] = '"'; - else if (**p == '\\') out[i++] = '\\'; - else if (**p == '/') out[i++] = '/'; + if (**p == 'n') + out[i++] = '\n'; + else if (**p == 't') + out[i++] = '\t'; + else if (**p == '"') + out[i++] = '"'; + else if (**p == '\\') + out[i++] = '\\'; + else if (**p == '/') + out[i++] = '/'; else if (**p == 'u') { (*p)++; unsigned int cp = 0; for (int j = 0; j < 4 && **p; j++, (*p)++) { cp <<= 4; char c = **p; - if (c >= '0' && c <= '9') cp |= (unsigned)(c - '0'); - else if (c >= 'a' && c <= 'f') cp |= (unsigned)(c - 'a' + 10); - else if (c >= 'A' && c <= 'F') cp |= (unsigned)(c - 'A' + 10); + if (c >= '0' && c <= '9') + cp |= (unsigned)(c - '0'); + else if (c >= 'a' && c <= 'f') + cp |= (unsigned)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') + cp |= (unsigned)(c - 'A' + 10); } if (cp < 0x80 && i + 1 < max_len) { out[i++] = (char)cp; @@ -159,7 +183,8 @@ static int parse_json_string(const char **p, char *out, size_t max_len) { (*p)++; } out[i] = '\0'; - if (**p != '"') return -1; + if (**p != '"') + return -1; (*p)++; return 0; } @@ -167,7 +192,10 @@ static int parse_json_string(const char **p, char *out, size_t max_len) { static int64_t parse_json_int(const char **p) { skip_ws(p); int neg = 0; - if (**p == '-') { neg = 1; (*p)++; } + if (**p == '-') { + neg = 1; + (*p)++; + } int64_t val = 0; while (**p >= '0' && **p <= '9') { val = val * 10 + (**p - '0'); @@ -211,23 +239,28 @@ static uint64_t fnv1a_hash_pair(const char *a, const char *b) { static int pair_key_eq(const char *key, const char *a, const char *b) { while (*a) { - if (*key++ != *a++) return 0; + if (*key++ != *a++) + return 0; } - if (*key++ != ' ') return 0; + if (*key++ != ' ') + return 0; while (*b) { - if (*key++ != *b++) return 0; + if (*key++ != *b++) + return 0; } return *key == '\0'; } static int next_pow2(int x) { int p = 1; - while (p < x) p <<= 1; + while (p < x) + p <<= 1; return p; } static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { - if (!map || cap <= 0 || !key) return -1; + if (!map || cap <= 0 || !key) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); for (int i = 0; i < cap; i++) { @@ -246,9 +279,11 @@ static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { } static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) { - if (!keyp) return -1; + if (!keyp) + return -1; char *key = *keyp; - if (!map || cap <= 0 || !key) return -1; + if (!map || cap <= 0 || !key) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); for (int i = 0; i < cap; i++) { @@ -268,26 +303,31 @@ static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int valu } static int map_get(const str_int_entry_t *map, int cap, const char *key) { - if (!map || cap <= 0 || !key) return -1; + if (!map || cap <= 0 || !key) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); for (int i = 0; i < cap; i++) { int idx = (pos + i) & mask; - if (!map[idx].key) return -1; - if (strcmp(map[idx].key, key) == 0) return map[idx].value; + if (!map[idx].key) + return -1; + if (strcmp(map[idx].key, key) == 0) + return map[idx].value; } return -1; } -static int map_get_pair(const str_int_entry_t *map, int cap, - const char *a, const char *b) { - if (!map || cap <= 0 || !a || !b) return -1; +static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, const char *b) { + if (!map || cap <= 0 || !a || !b) + return -1; int mask = cap - 1; int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask); for (int i = 0; i < cap; i++) { int idx = (pos + i) & mask; - if (!map[idx].key) return -1; - if (pair_key_eq(map[idx].key, a, b)) return map[idx].value; + if (!map[idx].key) + return -1; + if (pair_key_eq(map[idx].key, a, b)) + return map[idx].value; } return -1; } @@ -297,35 +337,43 @@ static int map_get_pair(const str_int_entry_t *map, int cap, * ======================================================================== */ static int utf8_char_len(unsigned char c) { - if ((c & 0x80) == 0) return 1; - if ((c & 0xE0) == 0xC0) return 2; - if ((c & 0xF0) == 0xE0) return 3; - if ((c & 0xF8) == 0xF0) return 4; + if ((c & 0x80) == 0) + return 1; + if ((c & 0xE0) == 0xC0) + return 2; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xF8) == 0xF0) + return 4; return 1; } static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { *out_syms = NULL; *out_n = 0; - if (!s || !*s) return 0; + if (!s || !*s) + return 0; int n = 0; - for (const unsigned char *p = (const unsigned char *)s; *p; ) { + for (const unsigned char *p = (const unsigned char *)s; *p;) { int l = utf8_char_len(*p); p += l; n++; } - if (n <= 0) return 0; + if (n <= 0) + return 0; char **syms = (char **)calloc((size_t)n, sizeof(char *)); - if (!syms) return -1; + if (!syms) + return -1; int i = 0; - for (const unsigned char *p = (const unsigned char *)s; *p; ) { + for (const unsigned char *p = (const unsigned char *)s; *p;) { int l = utf8_char_len(*p); syms[i] = (char *)malloc((size_t)l + 1); if (!syms[i]) { - for (int j = 0; j < i; j++) free(syms[j]); + for (int j = 0; j < i; j++) + free(syms[j]); free(syms); return -1; } @@ -343,7 +391,8 @@ static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { static char *str_concat2(const char *a, const char *b) { size_t la = strlen(a), lb = strlen(b); char *r = (char *)malloc(la + lb + 1); - if (!r) return NULL; + if (!r) + return NULL; memcpy(r, a, la); memcpy(r + la, b, lb); r[la + lb] = '\0'; @@ -355,21 +404,23 @@ static char *str_concat2(const char *a, const char *b) { * ======================================================================== */ typedef struct { - uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ + uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ int rank; int merged_id; } pair_merge_entry_t; static uint64_t pair_hash(uint64_t k) { /* splitmix64 finalizer */ - k ^= k >> 30; k *= 0xbf58476d1ce4e5b9ULL; - k ^= k >> 27; k *= 0x94d049bb133111ebULL; + k ^= k >> 30; + k *= 0xbf58476d1ce4e5b9ULL; + k ^= k >> 27; + k *= 0x94d049bb133111ebULL; k ^= k >> 31; return k; } -static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, - int rank, int merged_id) { +static void +pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) { int mask = cap - 1; int idx = (int)(pair_hash(key) & (uint64_t)mask); for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) { @@ -379,22 +430,21 @@ static void pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, map[idx].merged_id = merged_id; return; } - if (map[idx].key == key) return; /* first (lowest) rank wins */ + if (map[idx].key == key) + return; /* first (lowest) rank wins */ } } /* Returns the rank and sets *merged_id, or INT_MAX when the pair never * merges. */ -static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, - int *merged_id) { - const pair_merge_entry_t *map = - (const pair_merge_entry_t *)tok->int_merges; +static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, int *merged_id) { + const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges; uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right; int mask = tok->int_merges_cap - 1; int idx = (int)(pair_hash(key) & (uint64_t)mask); - for (int probe = 0; probe < tok->int_merges_cap; - probe++, idx = (idx + 1) & mask) { - if (map[idx].key == UINT64_MAX) return INT_MAX; + for (int probe = 0; probe < tok->int_merges_cap; probe++, idx = (idx + 1) & mask) { + if (map[idx].key == UINT64_MAX) + return INT_MAX; if (map[idx].key == key) { *merged_id = map[idx].merged_id; return map[idx].rank; @@ -404,10 +454,10 @@ static int pair_merge_lookup(const qwen_tokenizer_t *tok, int left, int right, } static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) { - if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX; + if (!tok->merge_map || tok->merge_map_cap <= 0) + return INT_MAX; - int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, - tok->merge_map_cap, a, b); + int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, a, b); return rank >= 0 ? rank : INT_MAX; } @@ -415,7 +465,8 @@ static int append_id(int **arr, int *n, int *cap, int id) { if (*n >= *cap) { int new_cap = (*cap == 0) ? 16 : (*cap * 2); int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int)); - if (!tmp) return -1; + if (!tmp) + return -1; *arr = tmp; *cap = new_cap; } @@ -429,25 +480,28 @@ static char *text_to_bpe_unicode(const char *text) { size_t len = strlen(text); /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */ char *out = (char *)malloc(len * 3 + 1); - if (!out) return NULL; + if (!out) + return NULL; size_t w = 0; for (size_t i = 0; i < len; i++) { int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; char tmp[4]; int n = utf8_encode_cp(cp, tmp); - for (int j = 0; j < n; j++) out[w++] = tmp[j]; + for (int j = 0; j < n; j++) + out[w++] = tmp[j]; } out[w] = '\0'; return out; } -static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, - int **out_ids, int *out_n); +static int +encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n); -static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, - int *cp_out, int *len_out) { - if (p >= end || *p == '\0') return 0; +static int +utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) { + if (p >= end || *p == '\0') + return 0; unsigned char c = *p; if ((c & 0x80) == 0) { @@ -455,27 +509,19 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, *len_out = 1; return 1; } - if ((c & 0xE0) == 0xC0 && p + 1 < end && - (p[1] & 0xC0) == 0x80) { + if ((c & 0xE0) == 0xC0 && p + 1 < end && (p[1] & 0xC0) == 0x80) { *cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F); *len_out = 2; return 1; } - if ((c & 0xF0) == 0xE0 && p + 2 < end && - (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { - *cp_out = ((c & 0x0F) << 12) | - ((p[1] & 0x3F) << 6) | - (p[2] & 0x3F); + if ((c & 0xF0) == 0xE0 && p + 2 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { + *cp_out = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F); *len_out = 3; return 1; } - if ((c & 0xF8) == 0xF0 && p + 3 < end && - (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && + if ((c & 0xF8) == 0xF0 && p + 3 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && (p[3] & 0xC0) == 0x80) { - *cp_out = ((c & 0x07) << 18) | - ((p[1] & 0x3F) << 12) | - ((p[2] & 0x3F) << 6) | - (p[3] & 0x3F); + *cp_out = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) | ((p[2] & 0x3F) << 6) | (p[3] & 0x3F); *len_out = 4; return 1; } @@ -494,47 +540,56 @@ static int utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, #define CPC_LETTER 4 static const unsigned char ascii_cp_class[128] = { - ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE, - ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE, - ['0' ... '9'] = CPC_NUMBER, - ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER, + ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE, + ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE, + ['0' ... '9'] = CPC_NUMBER, ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER, }; static int is_number_cp(int cp) { - if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_NUMBER; - if (cp >= 0x0660 && cp <= 0x0669) return 1; - if (cp >= 0x06F0 && cp <= 0x06F9) return 1; - if (cp >= 0x0966 && cp <= 0x096F) return 1; - if (cp >= 0xFF10 && cp <= 0xFF19) return 1; + if ((unsigned)cp < 0x80) + return ascii_cp_class[cp] & CPC_NUMBER; + if (cp >= 0x0660 && cp <= 0x0669) + return 1; + if (cp >= 0x06F0 && cp <= 0x06F9) + return 1; + if (cp >= 0x0966 && cp <= 0x096F) + return 1; + if (cp >= 0xFF10 && cp <= 0xFF19) + return 1; return 0; } static int is_space_cp(int cp) { - if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_SPACE; - return cp == 0x85 || cp == 0xA0 || - cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || - cp == 0x2028 || cp == 0x2029 || cp == 0x202F || - cp == 0x205F || cp == 0x3000; + if ((unsigned)cp < 0x80) + return ascii_cp_class[cp] & CPC_SPACE; + return cp == 0x85 || cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || + cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; } static int is_symbolish_cp(int cp) { - if (cp < 0x80) return 0; - if (cp >= 0x0300 && cp <= 0x036F) return 1; - if (cp >= 0x00A1 && cp <= 0x00BF && - cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) + if (cp < 0x80) + return 0; + if (cp >= 0x0300 && cp <= 0x036F) + return 1; + if (cp >= 0x00A1 && cp <= 0x00BF && cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) + return 1; + if (cp == 0x00D7 || cp == 0x00F7) + return 1; + if (cp >= 0x064B && cp <= 0x065F) + return 1; + if ((cp >= 0x0900 && cp <= 0x0903) || (cp >= 0x093A && cp <= 0x094F) || + (cp >= 0x0951 && cp <= 0x0957) || (cp >= 0x0962 && cp <= 0x0963)) return 1; - if (cp == 0x00D7 || cp == 0x00F7) return 1; - if (cp >= 0x064B && cp <= 0x065F) return 1; - if ((cp >= 0x0900 && cp <= 0x0903) || - (cp >= 0x093A && cp <= 0x094F) || - (cp >= 0x0951 && cp <= 0x0957) || - (cp >= 0x0962 && cp <= 0x0963)) + if (cp >= 0x2000 && cp <= 0x206F) + return 1; + if (cp >= 0x2190 && cp <= 0x27BF) + return 1; + if (cp >= 0x2E00 && cp <= 0x2E7F) + return 1; + if (cp >= 0x3000 && cp <= 0x303F) + return 1; + if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1; - if (cp >= 0x2000 && cp <= 0x206F) return 1; - if (cp >= 0x2190 && cp <= 0x27BF) return 1; - if (cp >= 0x2E00 && cp <= 0x2E7F) return 1; - if (cp >= 0x3000 && cp <= 0x303F) return 1; - if (cp >= 0x1F000 && cp <= 0x1FAFF) return 1; return 0; } @@ -542,63 +597,162 @@ static int compose_latin_mark(int base, int mark) { switch (mark) { case 0x0300: switch (base) { - case 'A': return 0x00C0; case 'E': return 0x00C8; - case 'I': return 0x00CC; case 'O': return 0x00D2; - case 'U': return 0x00D9; case 'a': return 0x00E0; - case 'e': return 0x00E8; case 'i': return 0x00EC; - case 'o': return 0x00F2; case 'u': return 0x00F9; + case 'A': + return 0x00C0; + case 'E': + return 0x00C8; + case 'I': + return 0x00CC; + case 'O': + return 0x00D2; + case 'U': + return 0x00D9; + case 'a': + return 0x00E0; + case 'e': + return 0x00E8; + case 'i': + return 0x00EC; + case 'o': + return 0x00F2; + case 'u': + return 0x00F9; } break; case 0x0301: switch (base) { - case 'A': return 0x00C1; case 'C': return 0x0106; - case 'E': return 0x00C9; case 'I': return 0x00CD; - case 'L': return 0x0139; case 'N': return 0x0143; - case 'O': return 0x00D3; case 'R': return 0x0154; - case 'S': return 0x015A; case 'U': return 0x00DA; - case 'Y': return 0x00DD; case 'Z': return 0x0179; - case 'a': return 0x00E1; case 'c': return 0x0107; - case 'e': return 0x00E9; case 'i': return 0x00ED; - case 'l': return 0x013A; case 'n': return 0x0144; - case 'o': return 0x00F3; case 'r': return 0x0155; - case 's': return 0x015B; case 'u': return 0x00FA; - case 'y': return 0x00FD; case 'z': return 0x017A; + case 'A': + return 0x00C1; + case 'C': + return 0x0106; + case 'E': + return 0x00C9; + case 'I': + return 0x00CD; + case 'L': + return 0x0139; + case 'N': + return 0x0143; + case 'O': + return 0x00D3; + case 'R': + return 0x0154; + case 'S': + return 0x015A; + case 'U': + return 0x00DA; + case 'Y': + return 0x00DD; + case 'Z': + return 0x0179; + case 'a': + return 0x00E1; + case 'c': + return 0x0107; + case 'e': + return 0x00E9; + case 'i': + return 0x00ED; + case 'l': + return 0x013A; + case 'n': + return 0x0144; + case 'o': + return 0x00F3; + case 'r': + return 0x0155; + case 's': + return 0x015B; + case 'u': + return 0x00FA; + case 'y': + return 0x00FD; + case 'z': + return 0x017A; } break; case 0x0302: switch (base) { - case 'A': return 0x00C2; case 'E': return 0x00CA; - case 'I': return 0x00CE; case 'O': return 0x00D4; - case 'U': return 0x00DB; case 'a': return 0x00E2; - case 'e': return 0x00EA; case 'i': return 0x00EE; - case 'o': return 0x00F4; case 'u': return 0x00FB; + case 'A': + return 0x00C2; + case 'E': + return 0x00CA; + case 'I': + return 0x00CE; + case 'O': + return 0x00D4; + case 'U': + return 0x00DB; + case 'a': + return 0x00E2; + case 'e': + return 0x00EA; + case 'i': + return 0x00EE; + case 'o': + return 0x00F4; + case 'u': + return 0x00FB; } break; case 0x0303: switch (base) { - case 'A': return 0x00C3; case 'N': return 0x00D1; - case 'O': return 0x00D5; case 'a': return 0x00E3; - case 'n': return 0x00F1; case 'o': return 0x00F5; + case 'A': + return 0x00C3; + case 'N': + return 0x00D1; + case 'O': + return 0x00D5; + case 'a': + return 0x00E3; + case 'n': + return 0x00F1; + case 'o': + return 0x00F5; } break; case 0x0308: switch (base) { - case 'A': return 0x00C4; case 'E': return 0x00CB; - case 'I': return 0x00CF; case 'O': return 0x00D6; - case 'U': return 0x00DC; case 'Y': return 0x0178; - case 'a': return 0x00E4; case 'e': return 0x00EB; - case 'i': return 0x00EF; case 'o': return 0x00F6; - case 'u': return 0x00FC; case 'y': return 0x00FF; + case 'A': + return 0x00C4; + case 'E': + return 0x00CB; + case 'I': + return 0x00CF; + case 'O': + return 0x00D6; + case 'U': + return 0x00DC; + case 'Y': + return 0x0178; + case 'a': + return 0x00E4; + case 'e': + return 0x00EB; + case 'i': + return 0x00EF; + case 'o': + return 0x00F6; + case 'u': + return 0x00FC; + case 'y': + return 0x00FF; } break; case 0x030A: switch (base) { - case 'A': return 0x00C5; case 'a': return 0x00E5; + case 'A': + return 0x00C5; + case 'a': + return 0x00E5; } break; case 0x0327: switch (base) { - case 'C': return 0x00C7; case 'c': return 0x00E7; + case 'C': + return 0x00C7; + case 'c': + return 0x00E7; } break; } @@ -609,13 +763,15 @@ static char *normalize_nfc_latin(const char *text) { const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); char *out = (char *)malloc((size_t)(end - p) * 2 + 1); - if (!out) return NULL; + if (!out) + return NULL; size_t w = 0; while (p < end) { const unsigned char *cur = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; p += len; if (p < end) { @@ -625,7 +781,8 @@ static char *normalize_nfc_latin(const char *text) { if (composed) { char tmp[4]; int n = utf8_encode_cp(composed, tmp); - for (int i = 0; i < n; i++) out[w++] = tmp[i]; + for (int i = 0; i < n; i++) + out[w++] = tmp[i]; p += mark_len; continue; } @@ -640,26 +797,22 @@ static char *normalize_nfc_latin(const char *text) { } static int is_letter_cp(int cp) { - if ((unsigned)cp < 0x80) return ascii_cp_class[cp] & CPC_LETTER; + if ((unsigned)cp < 0x80) + return ascii_cp_class[cp] & CPC_LETTER; if (is_space_cp(cp) || is_number_cp(cp) || is_symbolish_cp(cp)) return 0; return 1; } -static int is_word_cp(int cp) { - return is_letter_cp(cp) || is_number_cp(cp); -} +static int is_word_cp(int cp) { return is_letter_cp(cp) || is_number_cp(cp); } -static int is_newline_cp(int cp) { - return cp == '\n' || cp == '\r'; -} +static int is_newline_cp(int cp) { return cp == '\n' || cp == '\r'; } -static int ascii_lower(int c) { - return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; -} +static int ascii_lower(int c) { return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; } static int match_contraction(const unsigned char *p, const unsigned char *end) { - if (p >= end || *p != '\'') return 0; + if (p >= end || *p != '\'') + return 0; int remain = (int)(end - p); if (remain >= 2) { int c1 = ascii_lower(p[1]); @@ -669,27 +822,31 @@ static int match_contraction(const unsigned char *p, const unsigned char *end) { if (remain >= 3) { int c1 = ascii_lower(p[1]); int c2 = ascii_lower(p[2]); - if ((c1 == 'r' && c2 == 'e') || - (c1 == 'v' && c2 == 'e') || - (c1 == 'l' && c2 == 'l')) + if ((c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l')) return 3; } return 0; } static int append_encoded_piece(const qwen_tokenizer_t *tok, - const char *piece, size_t piece_len, - int **ids, int *n_ids, int *cap) { - if (piece_len == 0) return 0; + const char *piece, + size_t piece_len, + int **ids, + int *n_ids, + int *cap) { + if (piece_len == 0) + return 0; char *tmp = (char *)malloc(piece_len + 1); - if (!tmp) return -1; + if (!tmp) + return -1; memcpy(tmp, piece, piece_len); tmp[piece_len] = '\0'; char *mapped = text_to_bpe_unicode(tmp); free(tmp); - if (!mapped) return -1; + if (!mapped) + return -1; int *piece_ids = NULL; int n_piece = 0; @@ -710,8 +867,8 @@ static int append_encoded_piece(const qwen_tokenizer_t *tok, return 0; } -static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, - int **out_ids, int *out_n) { +static int +encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) { const unsigned char *start = (const unsigned char *)text; const unsigned char *p = start; const unsigned char *end = start + strlen(text); @@ -721,7 +878,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (p < end) { const unsigned char *piece = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; int clen = match_contraction(p, end); if (clen > 0) { @@ -731,13 +889,13 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, int next_cp = 0, next_len = 0; int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - if (!is_newline_cp(cp) && !is_word_cp(cp) && - has_next && is_letter_cp(next_cp)) { + if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_letter_cp(cp)) { @@ -745,7 +903,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_number_cp(cp)) { @@ -756,14 +915,17 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (q < end) { int c = 0, l = 0; utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) break; - if (is_newline_cp(c)) saw_newline = 1; + if (!is_space_cp(c)) + break; + if (is_newline_cp(c)) + saw_newline = 1; q += l; if (saw_newline) { while (q < end) { int c2 = 0, l2 = 0; utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) break; + if (!is_newline_cp(c2)) + break; q += l2; } break; @@ -772,19 +934,20 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, if (saw_newline) { p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && - !is_word_cp(next_cp)) { + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } else { @@ -794,7 +957,8 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (q2 < end) { int c = 0, l = 0; utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) break; + if (!is_space_cp(c)) + break; last = q2; q2 += l; count++; @@ -809,20 +973,22 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } } - if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), - &ids, &n_ids, &cap) != 0) { + if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids, + &cap) != 0) { free(ids); return -1; } @@ -834,14 +1000,17 @@ static int encode_pretokenized(const qwen_tokenizer_t *tok, const char *text, } /* Encode one mapped BPE unicode string to token IDs. */ -static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { +static int +encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { *out_ids = NULL; *out_n = 0; - if (!mapped || !*mapped) return 0; + if (!mapped || !*mapped) + return 0; char **syms = NULL; int n_syms = 0; - if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) return -1; + if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) + return -1; if (n_syms <= 0) { free(syms); return 0; @@ -857,18 +1026,21 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int best_i = i; } } - if (best_i < 0 || best_rank == INT_MAX) break; + if (best_i < 0 || best_rank == INT_MAX) + break; char *merged = str_concat2(syms[best_i], syms[best_i + 1]); if (!merged) { - for (int i = 0; i < n_syms; i++) free(syms[i]); + for (int i = 0; i < n_syms; i++) + free(syms[i]); free(syms); return -1; } free(syms[best_i]); free(syms[best_i + 1]); syms[best_i] = merged; - for (int j = best_i + 1; j < n_syms - 1; j++) syms[j] = syms[j + 1]; + for (int j = best_i + 1; j < n_syms - 1; j++) + syms[j] = syms[j + 1]; n_syms--; } @@ -878,20 +1050,23 @@ static int encode_bpe_word(const qwen_tokenizer_t *tok, const char *mapped, int int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]); if (id < 0) { /* Should not happen with valid vocab + merges + byte-level mapping. */ - for (int k = 0; k < n_syms; k++) free(syms[k]); + for (int k = 0; k < n_syms; k++) + free(syms[k]); free(syms); free(ids); return -1; } if (append_id(&ids, &n_ids, &cap, id) != 0) { - for (int k = 0; k < n_syms; k++) free(syms[k]); + for (int k = 0; k < n_syms; k++) + free(syms[k]); free(syms); free(ids); return -1; } } - for (int i = 0; i < n_syms; i++) free(syms[i]); + for (int i = 0; i < n_syms; i++) + free(syms[i]); free(syms); *out_ids = ids; @@ -927,72 +1102,87 @@ struct qwen_tokenizer_workspace { }; static int reserve_bpe(qwen_tokenizer_workspace_t *ws, int need) { - if (ws->bpe_cap >= need) return 0; + if (ws->bpe_cap >= need) + return 0; int new_cap = ws->bpe_cap ? ws->bpe_cap : 64; while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; + if (new_cap > INT_MAX / 2) + return -1; new_cap *= 2; } int *p; -#define R(field) \ +#define R(field) \ p = (int *)realloc(ws->field, (size_t)new_cap * sizeof(int)); \ - if (!p) return -1; \ + if (!p) \ + return -1; \ ws->field = p; - R(bpe_ids) R(bpe_prv) R(bpe_nxt) R(bpe_rnk) R(bpe_mrg) + R(bpe_ids) + R(bpe_prv) + R(bpe_nxt) + R(bpe_rnk) + R(bpe_mrg) #undef R - ws->bpe_cap = new_cap; + ws->bpe_cap = new_cap; return 0; } static int reserve_bytes(char **buf, size_t *cap, size_t need) { - if (*cap >= need) return 0; + if (*cap >= need) + return 0; size_t new_cap = *cap ? *cap : 256; while (new_cap < need) { - if (new_cap > SIZE_MAX / 2) return -1; + if (new_cap > SIZE_MAX / 2) + return -1; new_cap *= 2; } char *p = (char *)realloc(*buf, new_cap); - if (!p) return -1; + if (!p) + return -1; *buf = p; *cap = new_cap; return 0; } static int reserve_ints(int **buf, int *cap, int need) { - if (*cap >= need) return 0; + if (*cap >= need) + return 0; int new_cap = *cap ? *cap : 64; while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; + if (new_cap > INT_MAX / 2) + return -1; new_cap *= 2; } int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int)); - if (!p) return -1; + if (!p) + return -1; *buf = p; *cap = new_cap; return 0; } static int reserve_syms(qwen_tokenizer_workspace_t *ws, int need) { - if (ws->syms_cap >= need) return 0; + if (ws->syms_cap >= need) + return 0; int new_cap = ws->syms_cap ? ws->syms_cap : 64; while (new_cap < need) { - if (new_cap > INT_MAX / 2) return -1; + if (new_cap > INT_MAX / 2) + return -1; new_cap *= 2; } - bpe_sym_ref_t *p = - (bpe_sym_ref_t *)realloc(ws->syms, - (size_t)new_cap * sizeof(*ws->syms)); - if (!p) return -1; + bpe_sym_ref_t *p = (bpe_sym_ref_t *)realloc(ws->syms, (size_t)new_cap * sizeof(*ws->syms)); + if (!p) + return -1; ws->syms = p; ws->syms_cap = new_cap; return 0; } -static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, - size_t len, size_t *off) { - if (ws->arena_len > SIZE_MAX - len - 1) return -1; +static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) { + if (ws->arena_len > SIZE_MAX - len - 1) + return -1; size_t need = ws->arena_len + len + 1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) return -1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) + return -1; *off = ws->arena_len; memcpy(ws->arena + ws->arena_len, s, len); ws->arena_len += len; @@ -1000,26 +1190,26 @@ static int arena_append(qwen_tokenizer_workspace_t *ws, const char *s, return 0; } -static const char *arena_str(const qwen_tokenizer_workspace_t *ws, - bpe_sym_ref_t sym) { +static const char *arena_str(const qwen_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) { return ws->arena + sym.off; } -static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, - const char *text, const char **out) { +static int +normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, const char *text, const char **out) { const unsigned char *p = (const unsigned char *)text; const unsigned char *end = p + strlen(text); size_t in_len = (size_t)(end - p); - if (in_len > (SIZE_MAX - 1) / 2) return -1; - if (reserve_bytes(&ws->normalized, &ws->normalized_cap, - in_len * 2 + 1) != 0) + if (in_len > (SIZE_MAX - 1) / 2) + return -1; + if (reserve_bytes(&ws->normalized, &ws->normalized_cap, in_len * 2 + 1) != 0) return -1; size_t w = 0; while (p < end) { const unsigned char *cur = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; p += len; if (p < end) { @@ -1046,10 +1236,12 @@ static int normalize_nfc_latin_into(qwen_tokenizer_workspace_t *ws, } static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, - const char *text, size_t len, + const char *text, + size_t len, const char **out) { init_gpt2_mapping(); - if (len > (SIZE_MAX - 1) / 3) return -1; + if (len > (SIZE_MAX - 1) / 3) + return -1; if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0) return -1; @@ -1066,8 +1258,7 @@ static int text_to_bpe_unicode_into(qwen_tokenizer_workspace_t *ws, return 0; } -static int output_append_id(int *out_ids, int out_cap, int *n_ids, - int *overflow, int id) { +static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow, int id) { if (*n_ids < out_cap && out_ids) out_ids[*n_ids] = id; else @@ -1079,22 +1270,27 @@ static int output_append_id(int *out_ids, int out_cap, int *n_ids, static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *mapped, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (!mapped || !*mapped) return 0; + int *out_ids, + int out_cap, + int *n_ids, + int *overflow) { + if (!mapped || !*mapped) + return 0; int n_syms = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { int l = utf8_char_len(*p); p += l; n_syms++; } - if (n_syms <= 0) return 0; - if (reserve_syms(ws, n_syms) != 0) return -1; + if (n_syms <= 0) + return 0; + if (reserve_syms(ws, n_syms) != 0) + return -1; ws->arena_len = 0; int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { int l = utf8_char_len(*p); size_t off = 0; if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0) @@ -1107,21 +1303,21 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, int best_rank = INT_MAX; int best_i = -1; for (int j = 0; j < n_syms - 1; j++) { - int r = merge_rank(tok, arena_str(ws, ws->syms[j]), - arena_str(ws, ws->syms[j + 1])); + int r = merge_rank(tok, arena_str(ws, ws->syms[j]), arena_str(ws, ws->syms[j + 1])); if (r < best_rank) { best_rank = r; best_i = j; } } - if (best_i < 0 || best_rank == INT_MAX) break; + if (best_i < 0 || best_rank == INT_MAX) + break; const char *a = arena_str(ws, ws->syms[best_i]); const char *b = arena_str(ws, ws->syms[best_i + 1]); size_t la = strlen(a), lb = strlen(b); - if (la > SIZE_MAX - lb) return -1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, - ws->arena_len + la + lb + 1) != 0) + if (la > SIZE_MAX - lb) + return -1; + if (reserve_bytes(&ws->arena, &ws->arena_cap, ws->arena_len + la + lb + 1) != 0) return -1; a = arena_str(ws, ws->syms[best_i]); @@ -1139,9 +1335,10 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, } for (int j = 0; j < n_syms; j++) { - int id = map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, arena_str(ws, ws->syms[j])); - if (id < 0) return -1; + int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, + arena_str(ws, ws->syms[j])); + if (id < 0) + return -1; output_append_id(out_ids, out_cap, n_ids, overflow, id); } return 0; @@ -1155,35 +1352,41 @@ static int encode_bpe_word_into_slow(const qwen_tokenizer_t *tok, static int encode_bpe_word_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *mapped, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (!mapped || !*mapped) return 0; + int *out_ids, + int out_cap, + int *n_ids, + int *overflow) { + if (!mapped || !*mapped) + return 0; if (!tok->int_merges || !tok->cp_to_id) - return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, - n_ids, overflow); + return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); int n = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { p += utf8_char_len(*p); n++; } - if (n <= 0) return 0; - if (reserve_bpe(ws, n) != 0) return -1; + if (n <= 0) + return 0; + if (reserve_bpe(ws, n) != 0) + return -1; int *ids = ws->bpe_ids, *prv = ws->bpe_prv, *nxt = ws->bpe_nxt; int *rnk = ws->bpe_rnk, *mrg = ws->bpe_mrg; int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p; ) { + for (const unsigned char *p = (const unsigned char *)mapped; *p;) { int len = utf8_char_len(*p); int cp; - if (len == 1) cp = *p; - else if (len == 2) cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F); - else cp = 512; /* byte-level alphabet never exceeds two bytes */ + if (len == 1) + cp = *p; + else if (len == 2) + cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F); + else + cp = 512; /* byte-level alphabet never exceeds two bytes */ int id = cp < 512 ? tok->cp_to_id[cp] : -1; if (id < 0) - return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, - out_cap, n_ids, overflow); + return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); ids[i] = id; prv[i] = i - 1; nxt[i] = i + 1 < n ? i + 1 : -1; @@ -1193,7 +1396,8 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, for (int j = 0; j < n - 1; j++) rnk[j] = pair_merge_lookup(tok, ids[j], ids[j + 1], &mrg[j]); - if (n > 0) rnk[n - 1] = INT_MAX; + if (n > 0) + rnk[n - 1] = INT_MAX; for (;;) { int best = -1, best_rank = INT_MAX; @@ -1203,19 +1407,19 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, best = j; } } - if (best < 0) break; + if (best < 0) + break; ids[best] = mrg[best]; int dead = nxt[best]; nxt[best] = nxt[dead]; - if (nxt[dead] != -1) prv[nxt[dead]] = best; + if (nxt[dead] != -1) + prv[nxt[dead]] = best; if (prv[best] != -1) - rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], - &mrg[prv[best]]); - rnk[best] = nxt[best] != -1 - ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) - : INT_MAX; + rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]); + rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) + : INT_MAX; } for (int j = 0; j != -1; j = nxt[j]) @@ -1225,22 +1429,26 @@ static int encode_bpe_word_into(const qwen_tokenizer_t *tok, static int append_encoded_piece_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, - const char *piece, size_t piece_len, - int *out_ids, int out_cap, - int *n_ids, int *overflow) { - if (piece_len == 0) return 0; + const char *piece, + size_t piece_len, + int *out_ids, + int out_cap, + int *n_ids, + int *overflow) { + if (piece_len == 0) + return 0; const char *mapped = NULL; if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0) return -1; - return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, - overflow); + return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); } static int encode_pretokenized_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, - int *out_ids, int out_cap, + int *out_ids, + int out_cap, int *out_n) { const unsigned char *start = (const unsigned char *)text; const unsigned char *p = start; @@ -1251,7 +1459,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (p < end) { const unsigned char *piece = p; int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) break; + if (!utf8_decode_cp_len(p, end, &cp, &len)) + break; int clen = match_contraction(p, end); if (clen > 0) { @@ -1261,13 +1470,13 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, int next_cp = 0, next_len = 0; int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - if (!is_newline_cp(cp) && !is_word_cp(cp) && - has_next && is_letter_cp(next_cp)) { + if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_letter_cp(cp)) { @@ -1275,7 +1484,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) break; + if (!is_letter_cp(c)) + break; p += l; } } else if (is_number_cp(cp)) { @@ -1286,14 +1496,17 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (q < end) { int c = 0, l = 0; utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) break; - if (is_newline_cp(c)) saw_newline = 1; + if (!is_space_cp(c)) + break; + if (is_newline_cp(c)) + saw_newline = 1; q += l; if (saw_newline) { while (q < end) { int c2 = 0, l2 = 0; utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) break; + if (!is_newline_cp(c2)) + break; q += l2; } break; @@ -1302,19 +1515,20 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, if (saw_newline) { p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && - !is_word_cp(next_cp)) { + } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) { p = next + next_len; while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } else { @@ -1324,7 +1538,8 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (q2 < end) { int c = 0, l = 0; utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) break; + if (!is_space_cp(c)) + break; last = q2; q2 += l; count++; @@ -1339,21 +1554,22 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) break; + if (is_space_cp(c) || is_word_cp(c)) + break; p += l; } while (p < end) { int c = 0, l = 0; utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) break; + if (!is_newline_cp(c)) + break; p += l; } } } - if (append_encoded_piece_into(tok, ws, (const char *)piece, - (size_t)(p - piece), out_ids, out_cap, - &n_ids, &overflow) != 0) + if (append_encoded_piece_into(tok, ws, (const char *)piece, (size_t)(p - piece), out_ids, + out_cap, &n_ids, &overflow) != 0) return -1; } @@ -1364,11 +1580,13 @@ static int encode_pretokenized_into(const qwen_tokenizer_t *tok, static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { const char *slash = strrchr(vocab_path, '/'); if (!slash) { - if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) return -1; + if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) + return -1; return 0; } size_t dir_len = (size_t)(slash - vocab_path); - if (dir_len + 12 >= out_cap) return -1; + if (dir_len + 12 >= out_cap) + return -1; memcpy(out_path, vocab_path, dir_len); out_path[dir_len] = '\0'; snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt"); @@ -1377,22 +1595,28 @@ static int derive_merges_path(const char *vocab_path, char *out_path, size_t out static void trim_newline(char *line) { size_t n = strlen(line); - while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) line[--n] = '\0'; + while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) + line[--n] = '\0'; } static int parse_merge_pair(char *line, char **a, char **b) { *a = NULL; *b = NULL; trim_newline(line); - if (line[0] == '\0') return -1; - if (line[0] == '#') return -1; + if (line[0] == '\0') + return -1; + if (line[0] == '#') + return -1; char *sp = strchr(line, ' '); - if (!sp) return -1; + if (!sp) + return -1; *sp = '\0'; char *p2 = sp + 1; - while (*p2 == ' ') p2++; - if (*line == '\0' || *p2 == '\0') return -1; + while (*p2 == ' ') + p2++; + if (*line == '\0' || *p2 == '\0') + return -1; *a = line; *b = p2; return 0; @@ -1400,13 +1624,15 @@ static int parse_merge_pair(char *line, char **a, char **b) { static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { FILE *f = fopen(merges_path, "rb"); - if (!f) return -1; + if (!f) + return -1; int n_pairs = 0; char line[8192]; while (fgets(line, sizeof(line), f)) { char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) == 0) n_pairs++; + if (parse_merge_pair(line, &a, &b) == 0) + n_pairs++; } if (n_pairs <= 0) { fclose(f); @@ -1426,8 +1652,8 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { return -1; } tok->int_merges_cap = next_pow2(n_pairs * 2); - pair_merge_entry_t *pm = (pair_merge_entry_t *) - calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t)); + pair_merge_entry_t *pm = + (pair_merge_entry_t *)calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t)); if (pm) { for (int i = 0; i < tok->int_merges_cap; i++) { pm[i].key = UINT64_MAX; @@ -1440,11 +1666,13 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { int rank = 0; while (fgets(line, sizeof(line), f)) { char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) != 0) continue; + if (parse_merge_pair(line, &a, &b) != 0) + continue; size_t la = strlen(a), lb = strlen(b); char *key = (char *)malloc(la + 1 + lb + 1); - if (!key) continue; + if (!key) + continue; memcpy(key, a, la); key[la] = ' '; memcpy(key + la + 1, b, lb); @@ -1470,15 +1698,11 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { tok->int_merges = NULL; tok->int_merges_cap = 0; } else { - pair_map_insert((pair_merge_entry_t *)tok->int_merges, - tok->int_merges_cap, - ((uint64_t)(uint32_t)left << 32) | - (uint32_t)right, - rank, merged); + pair_map_insert((pair_merge_entry_t *)tok->int_merges, tok->int_merges_cap, + ((uint64_t)(uint32_t)left << 32) | (uint32_t)right, rank, merged); } } - (void)map_insert_owned((str_int_entry_t *)tok->merge_map, - tok->merge_map_cap, &key, rank); + (void)map_insert_owned((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, &key, rank); free(key); rank++; } @@ -1489,17 +1713,22 @@ static int load_merges_map(qwen_tokenizer_t *tok, const char *merges_path) { init_gpt2_mapping(); tok->cp_to_id = (int *)malloc(512 * sizeof(int)); if (tok->cp_to_id) { - for (int i = 0; i < 512; i++) tok->cp_to_id[i] = -1; + for (int i = 0; i < 512; i++) + tok->cp_to_id[i] = -1; for (int byte = 0; byte < 256 && tok->cp_to_id; byte++) { int cp = gpt2_byte_to_unicode[byte]; char buf[4]; int n; - if (cp < 0x80) { buf[0] = (char)cp; n = 1; } - else { buf[0] = (char)(0xC0 | (cp >> 6)); - buf[1] = (char)(0x80 | (cp & 0x3F)); n = 2; } + if (cp < 0x80) { + buf[0] = (char)cp; + n = 1; + } else { + buf[0] = (char)(0xC0 | (cp >> 6)); + buf[1] = (char)(0x80 | (cp & 0x3F)); + n = 2; + } buf[n] = '\0'; - int id = map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, buf); + int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, buf); if (id < 0) { free(tok->cp_to_id); tok->cp_to_id = NULL; @@ -1554,16 +1783,23 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { const char *saved = p; while (*p && *p != '}') { skip_ws(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) break; + if (parse_json_string(&p, key, sizeof(key)) != 0) + break; skip_ws(&p); - if (*p != ':') break; + if (*p != ':') + break; p++; int64_t id = parse_json_int(&p); - if (id > max_id) max_id = (int)id; + if (id > max_id) + max_id = (int)id; } int vocab_size = max_id + 1; @@ -1584,13 +1820,19 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { p = saved; while (*p && *p != '}') { skip_ws(&p); - if (*p == ',') { p++; continue; } - if (*p == '}') break; + if (*p == ',') { + p++; + continue; + } + if (*p == '}') + break; char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) break; + if (parse_json_string(&p, key, sizeof(key)) != 0) + break; skip_ws(&p); - if (*p != ':') break; + if (*p != ':') + break; p++; int64_t id = parse_json_int(&p); @@ -1604,7 +1846,9 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { free(json); int n_vocab_entries = 0; - for (int i = 0; i < vocab_size; i++) if (tok->id_to_bpe[i]) n_vocab_entries++; + for (int i = 0; i < vocab_size; i++) + if (tok->id_to_bpe[i]) + n_vocab_entries++; tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); if (!tok->vocab_map) { @@ -1620,7 +1864,8 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { char merges_path[1024]; if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { if (load_merges_map(tok, merges_path) != 0 && qwen_verbose >= 2) { - fprintf(stderr, "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", + fprintf(stderr, + "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", merges_path); } } @@ -1629,22 +1874,26 @@ qwen_tokenizer_t *qwen_tokenizer_load(const char *vocab_json_path) { } const char *qwen_tokenizer_decode(const qwen_tokenizer_t *tok, int token_id) { - if (!tok || token_id < 0 || token_id >= tok->vocab_size) return ""; + if (!tok || token_id < 0 || token_id >= tok->vocab_size) + return ""; return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; } int qwen_tokenizer_token_id(const qwen_tokenizer_t *tok, const char *token) { - if (!tok || !token || !tok->vocab_map) return -1; - return map_get((const str_int_entry_t *)tok->vocab_map, - tok->vocab_map_cap, token); + if (!tok || !token || !tok->vocab_map) + return -1; + return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token); } int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !text || text[0] == '\0') return NULL; + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !text || text[0] == '\0') + return NULL; char *normalized = normalize_nfc_latin(text); - if (!normalized) return NULL; + if (!normalized) + return NULL; int *ids = NULL; int n_ids = 0; @@ -1655,17 +1904,18 @@ int *qwen_tokenizer_encode(const qwen_tokenizer_t *tok, const char *text, int *o } free(normalized); - if (out_n_tokens) *out_n_tokens = n_ids; + if (out_n_tokens) + *out_n_tokens = n_ids; return ids; } qwen_tokenizer_workspace_t *qwen_tokenizer_workspace_new(void) { - return (qwen_tokenizer_workspace_t *)calloc(1, - sizeof(qwen_tokenizer_workspace_t)); + return (qwen_tokenizer_workspace_t *)calloc(1, sizeof(qwen_tokenizer_workspace_t)); } void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { - if (!ws) return; + if (!ws) + return; free(ws->normalized); free(ws->mapped); free(ws->arena); @@ -1682,20 +1932,24 @@ void qwen_tokenizer_workspace_free(qwen_tokenizer_workspace_t *ws) { int qwen_tokenizer_encode_into(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, - int *out_ids, int out_cap, + int *out_ids, + int out_cap, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !ws || !text || out_cap < 0) return -1; - if (text[0] == '\0') return 0; + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text || out_cap < 0) + return -1; + if (text[0] == '\0') + return 0; const char *normalized = NULL; if (normalize_nfc_latin_into(ws, text, &normalized) != 0) return -1; int n_ids = 0; - int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, - &n_ids); - if (out_n_tokens) *out_n_tokens = n_ids; + int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, &n_ids); + if (out_n_tokens) + *out_n_tokens = n_ids; return rc; } @@ -1703,50 +1957,58 @@ int *qwen_tokenizer_encode_with_workspace(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, int *out_n_tokens) { - if (out_n_tokens) *out_n_tokens = 0; - if (!tok || !ws || !text || text[0] == '\0') return NULL; + if (out_n_tokens) + *out_n_tokens = 0; + if (!tok || !ws || !text || text[0] == '\0') + return NULL; size_t len = strlen(text); size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8; - if (cap_sz > (size_t)INT_MAX) return NULL; + if (cap_sz > (size_t)INT_MAX) + return NULL; int cap = (int)cap_sz; if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0) return NULL; int n_ids = 0; - int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, - &n_ids); + int rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); if (rc == -2) { if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) return NULL; - rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, - &n_ids); + rc = qwen_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); } - if (rc != 0 || n_ids <= 0) return NULL; + if (rc != 0 || n_ids <= 0) + return NULL; int *ids = (int *)malloc((size_t)n_ids * sizeof(int)); - if (!ids) return NULL; + if (!ids) + return NULL; memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int)); - if (out_n_tokens) *out_n_tokens = n_ids; + if (out_n_tokens) + *out_n_tokens = n_ids; return ids; } void qwen_tokenizer_free(qwen_tokenizer_t *tok) { - if (!tok) return; + if (!tok) + return; if (tok->id_to_text) { - for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_text[i]); + for (int i = 0; i < tok->vocab_size; i++) + free(tok->id_to_text[i]); free(tok->id_to_text); } if (tok->id_to_bpe) { - for (int i = 0; i < tok->vocab_size; i++) free(tok->id_to_bpe[i]); + for (int i = 0; i < tok->vocab_size; i++) + free(tok->id_to_bpe[i]); free(tok->id_to_bpe); } if (tok->merge_map) { str_int_entry_t *m = (str_int_entry_t *)tok->merge_map; - for (int i = 0; i < tok->merge_map_cap; i++) free(m[i].key); + for (int i = 0; i < tok->merge_map_cap; i++) + free(m[i].key); free(tok->merge_map); } free(tok->vocab_map); |
