diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /src/tokenizer_bpe.c | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'src/tokenizer_bpe.c')
| -rw-r--r-- | src/tokenizer_bpe.c | 2018 |
1 files changed, 0 insertions, 2018 deletions
diff --git a/src/tokenizer_bpe.c b/src/tokenizer_bpe.c deleted file mode 100644 index 511b29e..0000000 --- a/src/tokenizer_bpe.c +++ /dev/null @@ -1,2018 +0,0 @@ -#include "tokenizer_bpe.h" -#include "kernels.h" -#include <limits.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -/* ======================================================================== - * GPT-2 Bytes-to-Unicode Mapping - * ======================================================================== */ - -static int gpt2_byte_to_unicode[256]; -static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */ -static int gpt2_mapping_initialized = 0; - -static void init_gpt2_mapping(void) { - if (gpt2_mapping_initialized) - return; - - memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte)); - - int n = 0; - for (int b = 0; b < 256; b++) { - int is_normal = 0; - if (b >= 33 && b <= 126) - is_normal = 1; /* '!'..'~' */ - if (b >= 161 && b <= 172) - is_normal = 1; /* '¡'..'¬' */ - if (b >= 174 && b <= 255) - is_normal = 1; /* '®'..'ÿ' */ - - if (is_normal) - gpt2_byte_to_unicode[b] = b; - else - gpt2_byte_to_unicode[b] = 256 + n++; - } - - for (int b = 0; b < 256; b++) { - int cp = gpt2_byte_to_unicode[b]; - if (cp < 512) - gpt2_unicode_to_byte[cp] = b; - } - - gpt2_mapping_initialized = 1; -} - -/* Convert one Unicode codepoint to UTF-8 bytes. Returns bytes written. */ -static int utf8_encode_cp(int cp, char out[4]) { - if (cp < 0x80) { - out[0] = (char)cp; - return 1; - } - if (cp < 0x800) { - out[0] = (char)(0xC0 | (cp >> 6)); - out[1] = (char)(0x80 | (cp & 0x3F)); - return 2; - } - out[0] = (char)(0xE0 | (cp >> 12)); - out[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); - out[2] = (char)(0x80 | (cp & 0x3F)); - return 3; -} - -/* - * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text. - * Returns allocated string (caller must free). - */ -static char *decode_gpt2_token(const char *token_str) { - init_gpt2_mapping(); - - size_t len = strlen(token_str); - unsigned char *bytes = (unsigned char *)malloc(len + 1); - if (!bytes) - return NULL; - - int byte_count = 0; - const unsigned char *p = (const unsigned char *)token_str; - const unsigned char *end = p + len; - - while (p < end) { - int cp = 0; - int nbytes = 1; - - if ((*p & 0x80) == 0) { - cp = *p; - nbytes = 1; - } else if ((*p & 0xE0) == 0xC0) { - cp = (*p & 0x1F) << 6; - if (p + 1 < end) - cp |= (p[1] & 0x3F); - nbytes = 2; - } else if ((*p & 0xF0) == 0xE0) { - cp = (*p & 0x0F) << 12; - if (p + 1 < end) - cp |= (p[1] & 0x3F) << 6; - if (p + 2 < end) - cp |= (p[2] & 0x3F); - nbytes = 3; - } else if ((*p & 0xF8) == 0xF0) { - cp = (*p & 0x07) << 18; - if (p + 1 < end) - cp |= (p[1] & 0x3F) << 12; - if (p + 2 < end) - cp |= (p[2] & 0x3F) << 6; - if (p + 3 < end) - cp |= (p[3] & 0x3F); - nbytes = 4; - } else { - cp = *p; - nbytes = 1; - } - p += nbytes; - - if (cp < 512 && gpt2_unicode_to_byte[cp] >= 0) { - bytes[byte_count++] = (unsigned char)gpt2_unicode_to_byte[cp]; - } else { - bytes[byte_count++] = '?'; - } - } - bytes[byte_count] = '\0'; - return (char *)bytes; -} - -/* ======================================================================== - * Simple JSON parser for vocab.json - * ======================================================================== */ - -static void skip_ws(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') - (*p)++; -} - -static int parse_json_string(const char **p, char *out, size_t max_len) { - skip_ws(p); - if (**p != '"') - return -1; - (*p)++; - size_t i = 0; - while (**p && **p != '"' && i < max_len - 1) { - if (**p == '\\') { - (*p)++; - if (**p == 'n') - out[i++] = '\n'; - else if (**p == 't') - out[i++] = '\t'; - else if (**p == '"') - out[i++] = '"'; - else if (**p == '\\') - out[i++] = '\\'; - else if (**p == '/') - out[i++] = '/'; - else if (**p == 'u') { - (*p)++; - unsigned int cp = 0; - for (int j = 0; j < 4 && **p; j++, (*p)++) { - cp <<= 4; - char c = **p; - if (c >= '0' && c <= '9') - cp |= (unsigned)(c - '0'); - else if (c >= 'a' && c <= 'f') - cp |= (unsigned)(c - 'a' + 10); - else if (c >= 'A' && c <= 'F') - cp |= (unsigned)(c - 'A' + 10); - } - if (cp < 0x80 && i + 1 < max_len) { - out[i++] = (char)cp; - } else if (cp < 0x800 && i + 2 < max_len) { - out[i++] = (char)(0xC0 | (cp >> 6)); - out[i++] = (char)(0x80 | (cp & 0x3F)); - } else if (i + 3 < max_len) { - out[i++] = (char)(0xE0 | (cp >> 12)); - out[i++] = (char)(0x80 | ((cp >> 6) & 0x3F)); - out[i++] = (char)(0x80 | (cp & 0x3F)); - } - continue; - } else { - out[i++] = **p; - } - } else { - out[i++] = **p; - } - (*p)++; - } - out[i] = '\0'; - if (**p != '"') - return -1; - (*p)++; - return 0; -} - -static int64_t parse_json_int(const char **p) { - skip_ws(p); - int neg = 0; - if (**p == '-') { - neg = 1; - (*p)++; - } - int64_t val = 0; - while (**p >= '0' && **p <= '9') { - val = val * 10 + (**p - '0'); - (*p)++; - } - return neg ? -val : val; -} - -/* ======================================================================== - * String->int hash map (open addressing) - * ======================================================================== */ - -typedef struct { - char *key; /* NULL means empty slot */ - int value; -} str_int_entry_t; - -static uint64_t fnv1a_hash(const char *s) { - uint64_t h = 1469598103934665603ULL; - while (*s) { - h ^= (unsigned char)*s++; - h *= 1099511628211ULL; - } - return h; -} - -static uint64_t fnv1a_hash_pair(const char *a, const char *b) { - uint64_t h = 1469598103934665603ULL; - while (*a) { - h ^= (unsigned char)*a++; - h *= 1099511628211ULL; - } - h ^= (unsigned char)' '; - h *= 1099511628211ULL; - while (*b) { - h ^= (unsigned char)*b++; - h *= 1099511628211ULL; - } - return h; -} - -static int pair_key_eq(const char *key, const char *a, const char *b) { - while (*a) { - if (*key++ != *a++) - return 0; - } - if (*key++ != ' ') - return 0; - while (*b) { - if (*key++ != *b++) - return 0; - } - return *key == '\0'; -} - -static int next_pow2(int x) { - int p = 1; - while (p < x) - p <<= 1; - return p; -} - -static int map_insert(str_int_entry_t *map, int cap, char *key, int value) { - if (!map || cap <= 0 || !key) - return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) { - map[idx].key = key; - map[idx].value = value; - return 0; - } - if (strcmp(map[idx].key, key) == 0) { - map[idx].value = value; - return 0; - } - } - return -1; -} - -static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) { - if (!keyp) - return -1; - char *key = *keyp; - if (!map || cap <= 0 || !key) - return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) { - map[idx].key = key; - map[idx].value = value; - *keyp = NULL; - return 0; - } - if (strcmp(map[idx].key, key) == 0) { - map[idx].value = value; - return 0; - } - } - return -1; -} - -static int map_get(const str_int_entry_t *map, int cap, const char *key) { - if (!map || cap <= 0 || !key) - return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash(key) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) - return -1; - if (strcmp(map[idx].key, key) == 0) - return map[idx].value; - } - return -1; -} - -static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, const char *b) { - if (!map || cap <= 0 || !a || !b) - return -1; - int mask = cap - 1; - int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask); - for (int i = 0; i < cap; i++) { - int idx = (pos + i) & mask; - if (!map[idx].key) - return -1; - if (pair_key_eq(map[idx].key, a, b)) - return map[idx].value; - } - return -1; -} - -/* ======================================================================== - * BPE helpers - * ======================================================================== */ - -static int utf8_char_len(unsigned char c) { - if ((c & 0x80) == 0) - return 1; - if ((c & 0xE0) == 0xC0) - return 2; - if ((c & 0xF0) == 0xE0) - return 3; - if ((c & 0xF8) == 0xF0) - return 4; - return 1; -} - -static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) { - *out_syms = NULL; - *out_n = 0; - if (!s || !*s) - return 0; - - int n = 0; - for (const unsigned char *p = (const unsigned char *)s; *p;) { - int l = utf8_char_len(*p); - p += l; - n++; - } - if (n <= 0) - return 0; - - char **syms = (char **)calloc((size_t)n, sizeof(char *)); - if (!syms) - return -1; - - int i = 0; - for (const unsigned char *p = (const unsigned char *)s; *p;) { - int l = utf8_char_len(*p); - syms[i] = (char *)malloc((size_t)l + 1); - if (!syms[i]) { - for (int j = 0; j < i; j++) - free(syms[j]); - free(syms); - return -1; - } - memcpy(syms[i], p, (size_t)l); - syms[i][l] = '\0'; - p += l; - i++; - } - - *out_syms = syms; - *out_n = n; - return 0; -} - -static char *str_concat2(const char *a, const char *b) { - size_t la = strlen(a), lb = strlen(b); - char *r = (char *)malloc(la + lb + 1); - if (!r) - return NULL; - memcpy(r, a, la); - memcpy(r + la, b, lb); - r[la + lb] = '\0'; - return r; -} - -/* ======================================================================== - * Integer-pair BPE merges: (left_id, right_id) -> (rank, merged_id) - * ======================================================================== */ - -typedef struct { - uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */ - int rank; - int merged_id; -} pair_merge_entry_t; - -static uint64_t pair_hash(uint64_t k) { - /* splitmix64 finalizer */ - k ^= k >> 30; - k *= 0xbf58476d1ce4e5b9ULL; - k ^= k >> 27; - k *= 0x94d049bb133111ebULL; - k ^= k >> 31; - return k; -} - -static void -pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) { - int mask = cap - 1; - int idx = (int)(pair_hash(key) & (uint64_t)mask); - for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) { - if (map[idx].key == UINT64_MAX) { - map[idx].key = key; - map[idx].rank = rank; - map[idx].merged_id = merged_id; - return; - } - if (map[idx].key == key) - return; /* first (lowest) rank wins */ - } -} - -/* Returns the rank and sets *merged_id, or INT_MAX when the pair never - * merges. */ -static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right, int *merged_id) { - const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges; - uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right; - int mask = tok->int_merges_cap - 1; - int idx = (int)(pair_hash(key) & (uint64_t)mask); - for (int probe = 0; probe < tok->int_merges_cap; probe++, idx = (idx + 1) & mask) { - if (map[idx].key == UINT64_MAX) - return INT_MAX; - if (map[idx].key == key) { - *merged_id = map[idx].merged_id; - return map[idx].rank; - } - } - return INT_MAX; -} - -static int merge_rank(const embed_tokenizer_t *tok, const char *a, const char *b) { - if (!tok->merge_map || tok->merge_map_cap <= 0) - return INT_MAX; - - int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, a, b); - return rank >= 0 ? rank : INT_MAX; -} - -static int append_id(int **arr, int *n, int *cap, int id) { - if (*n >= *cap) { - int new_cap = (*cap == 0) ? 16 : (*cap * 2); - int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int)); - if (!tmp) - return -1; - *arr = tmp; - *cap = new_cap; - } - (*arr)[(*n)++] = id; - return 0; -} - -/* Convert UTF-8 bytes to GPT-2 byte-level unicode string. */ -static char *text_to_bpe_unicode(const char *text) { - init_gpt2_mapping(); - size_t len = strlen(text); - /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */ - char *out = (char *)malloc(len * 3 + 1); - if (!out) - return NULL; - - size_t w = 0; - for (size_t i = 0; i < len; i++) { - int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; - char tmp[4]; - int n = utf8_encode_cp(cp, tmp); - for (int j = 0; j < n; j++) - out[w++] = tmp[j]; - } - out[w] = '\0'; - return out; -} - -static int -encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n); - -static int -utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) { - if (p >= end || *p == '\0') - return 0; - - unsigned char c = *p; - if ((c & 0x80) == 0) { - *cp_out = c; - *len_out = 1; - return 1; - } - if ((c & 0xE0) == 0xC0 && p + 1 < end && (p[1] & 0xC0) == 0x80) { - *cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F); - *len_out = 2; - return 1; - } - if ((c & 0xF0) == 0xE0 && p + 2 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) { - *cp_out = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F); - *len_out = 3; - return 1; - } - if ((c & 0xF8) == 0xF0 && p + 3 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 && - (p[3] & 0xC0) == 0x80) { - *cp_out = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) | ((p[2] & 0x3F) << 6) | (p[3] & 0x3F); - *len_out = 4; - return 1; - } - - *cp_out = c; - *len_out = 1; - return 1; -} - -/* The pretokenizer classifies every codepoint, several times each, and the - * input is overwhelmingly ASCII; one table lookup replaces the range chains - * for cp < 0x80. Non-ASCII codepoints fall through to the full checks. - * ASCII has no symbolish codepoints, so that class needs no bit. */ -#define CPC_SPACE 1 -#define CPC_NUMBER 2 -#define CPC_LETTER 4 - -static const unsigned char ascii_cp_class[128] = { - ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE, - ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE, - ['0' ... '9'] = CPC_NUMBER, ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER, -}; - -static int is_number_cp(int cp) { - if ((unsigned)cp < 0x80) - return ascii_cp_class[cp] & CPC_NUMBER; - if (cp >= 0x0660 && cp <= 0x0669) - return 1; - if (cp >= 0x06F0 && cp <= 0x06F9) - return 1; - if (cp >= 0x0966 && cp <= 0x096F) - return 1; - if (cp >= 0xFF10 && cp <= 0xFF19) - return 1; - return 0; -} - -static int is_space_cp(int cp) { - if ((unsigned)cp < 0x80) - return ascii_cp_class[cp] & CPC_SPACE; - return cp == 0x85 || cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || - cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000; -} - -static int is_symbolish_cp(int cp) { - if (cp < 0x80) - return 0; - if (cp >= 0x0300 && cp <= 0x036F) - return 1; - if (cp >= 0x00A1 && cp <= 0x00BF && cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA) - return 1; - if (cp == 0x00D7 || cp == 0x00F7) - return 1; - if (cp >= 0x064B && cp <= 0x065F) - return 1; - if ((cp >= 0x0900 && cp <= 0x0903) || (cp >= 0x093A && cp <= 0x094F) || - (cp >= 0x0951 && cp <= 0x0957) || (cp >= 0x0962 && cp <= 0x0963)) - return 1; - if (cp >= 0x2000 && cp <= 0x206F) - return 1; - if (cp >= 0x2190 && cp <= 0x27BF) - return 1; - if (cp >= 0x2E00 && cp <= 0x2E7F) - return 1; - if (cp >= 0x3000 && cp <= 0x303F) - return 1; - if (cp >= 0x1F000 && cp <= 0x1FAFF) - return 1; - return 0; -} - -static int compose_latin_mark(int base, int mark) { - switch (mark) { - case 0x0300: - switch (base) { - case 'A': - return 0x00C0; - case 'E': - return 0x00C8; - case 'I': - return 0x00CC; - case 'O': - return 0x00D2; - case 'U': - return 0x00D9; - case 'a': - return 0x00E0; - case 'e': - return 0x00E8; - case 'i': - return 0x00EC; - case 'o': - return 0x00F2; - case 'u': - return 0x00F9; - } - break; - case 0x0301: - switch (base) { - case 'A': - return 0x00C1; - case 'C': - return 0x0106; - case 'E': - return 0x00C9; - case 'I': - return 0x00CD; - case 'L': - return 0x0139; - case 'N': - return 0x0143; - case 'O': - return 0x00D3; - case 'R': - return 0x0154; - case 'S': - return 0x015A; - case 'U': - return 0x00DA; - case 'Y': - return 0x00DD; - case 'Z': - return 0x0179; - case 'a': - return 0x00E1; - case 'c': - return 0x0107; - case 'e': - return 0x00E9; - case 'i': - return 0x00ED; - case 'l': - return 0x013A; - case 'n': - return 0x0144; - case 'o': - return 0x00F3; - case 'r': - return 0x0155; - case 's': - return 0x015B; - case 'u': - return 0x00FA; - case 'y': - return 0x00FD; - case 'z': - return 0x017A; - } - break; - case 0x0302: - switch (base) { - case 'A': - return 0x00C2; - case 'E': - return 0x00CA; - case 'I': - return 0x00CE; - case 'O': - return 0x00D4; - case 'U': - return 0x00DB; - case 'a': - return 0x00E2; - case 'e': - return 0x00EA; - case 'i': - return 0x00EE; - case 'o': - return 0x00F4; - case 'u': - return 0x00FB; - } - break; - case 0x0303: - switch (base) { - case 'A': - return 0x00C3; - case 'N': - return 0x00D1; - case 'O': - return 0x00D5; - case 'a': - return 0x00E3; - case 'n': - return 0x00F1; - case 'o': - return 0x00F5; - } - break; - case 0x0308: - switch (base) { - case 'A': - return 0x00C4; - case 'E': - return 0x00CB; - case 'I': - return 0x00CF; - case 'O': - return 0x00D6; - case 'U': - return 0x00DC; - case 'Y': - return 0x0178; - case 'a': - return 0x00E4; - case 'e': - return 0x00EB; - case 'i': - return 0x00EF; - case 'o': - return 0x00F6; - case 'u': - return 0x00FC; - case 'y': - return 0x00FF; - } - break; - case 0x030A: - switch (base) { - case 'A': - return 0x00C5; - case 'a': - return 0x00E5; - } - break; - case 0x0327: - switch (base) { - case 'C': - return 0x00C7; - case 'c': - return 0x00E7; - } - break; - } - return 0; -} - -static char *normalize_nfc_latin(const char *text) { - const unsigned char *p = (const unsigned char *)text; - const unsigned char *end = p + strlen(text); - char *out = (char *)malloc((size_t)(end - p) * 2 + 1); - if (!out) - return NULL; - - size_t w = 0; - while (p < end) { - const unsigned char *cur = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) - break; - p += len; - - if (p < end) { - int mark = 0, mark_len = 0; - utf8_decode_cp_len(p, end, &mark, &mark_len); - int composed = compose_latin_mark(cp, mark); - if (composed) { - char tmp[4]; - int n = utf8_encode_cp(composed, tmp); - for (int i = 0; i < n; i++) - out[w++] = tmp[i]; - p += mark_len; - continue; - } - } - - memcpy(out + w, cur, (size_t)len); - w += (size_t)len; - } - - out[w] = '\0'; - return out; -} - -static int is_letter_cp(int cp) { - if ((unsigned)cp < 0x80) - return ascii_cp_class[cp] & CPC_LETTER; - if (is_space_cp(cp) || is_number_cp(cp) || is_symbolish_cp(cp)) - return 0; - return 1; -} - -static int is_word_cp(int cp) { return is_letter_cp(cp) || is_number_cp(cp); } - -static int is_newline_cp(int cp) { return cp == '\n' || cp == '\r'; } - -static int ascii_lower(int c) { return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; } - -static int match_contraction(const unsigned char *p, const unsigned char *end) { - if (p >= end || *p != '\'') - return 0; - int remain = (int)(end - p); - if (remain >= 2) { - int c1 = ascii_lower(p[1]); - if (c1 == 's' || c1 == 't' || c1 == 'm' || c1 == 'd') - return 2; - } - if (remain >= 3) { - int c1 = ascii_lower(p[1]); - int c2 = ascii_lower(p[2]); - if ((c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l')) - return 3; - } - return 0; -} - -static int append_encoded_piece(const embed_tokenizer_t *tok, - const char *piece, - size_t piece_len, - int **ids, - int *n_ids, - int *cap) { - if (piece_len == 0) - return 0; - - char *tmp = (char *)malloc(piece_len + 1); - if (!tmp) - return -1; - memcpy(tmp, piece, piece_len); - tmp[piece_len] = '\0'; - - char *mapped = text_to_bpe_unicode(tmp); - free(tmp); - if (!mapped) - return -1; - - int *piece_ids = NULL; - int n_piece = 0; - if (encode_bpe_word(tok, mapped, &piece_ids, &n_piece) != 0) { - free(mapped); - free(piece_ids); - return -1; - } - free(mapped); - - for (int i = 0; i < n_piece; i++) { - if (append_id(ids, n_ids, cap, piece_ids[i]) != 0) { - free(piece_ids); - return -1; - } - } - free(piece_ids); - return 0; -} - -static int -encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) { - const unsigned char *start = (const unsigned char *)text; - const unsigned char *p = start; - const unsigned char *end = start + strlen(text); - int *ids = NULL; - int n_ids = 0, cap = 0; - - while (p < end) { - const unsigned char *piece = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) - break; - - int clen = match_contraction(p, end); - if (clen > 0) { - p += clen; - } else { - const unsigned char *next = p + len; - int next_cp = 0, next_len = 0; - int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - - if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) - break; - p += l; - } - } else if (is_letter_cp(cp)) { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) - break; - p += l; - } - } else if (is_number_cp(cp)) { - p += len; - } else if (is_space_cp(cp)) { - const unsigned char *q = p; - int saw_newline = 0; - while (q < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) - break; - if (is_newline_cp(c)) - saw_newline = 1; - q += l; - if (saw_newline) { - while (q < end) { - int c2 = 0, l2 = 0; - utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) - break; - q += l2; - } - break; - } - } - - if (saw_newline) { - p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) - break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) - break; - p += l; - } - } else { - const unsigned char *last = p; - const unsigned char *q2 = p; - int count = 0; - while (q2 < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) - break; - last = q2; - q2 += l; - count++; - } - if (q2 < end && count > 1) - p = last; - else - p = q2; - } - } else { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) - break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) - break; - p += l; - } - } - } - - if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids, - &cap) != 0) { - free(ids); - return -1; - } - } - - *out_ids = ids; - *out_n = n_ids; - return 0; -} - -/* Encode one mapped BPE unicode string to token IDs. */ -static int -encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) { - *out_ids = NULL; - *out_n = 0; - if (!mapped || !*mapped) - return 0; - - char **syms = NULL; - int n_syms = 0; - if (split_utf8_symbols(mapped, &syms, &n_syms) != 0) - return -1; - if (n_syms <= 0) { - free(syms); - return 0; - } - - while (n_syms > 1) { - int best_rank = INT_MAX; - int best_i = -1; - for (int i = 0; i < n_syms - 1; i++) { - int r = merge_rank(tok, syms[i], syms[i + 1]); - if (r < best_rank) { - best_rank = r; - best_i = i; - } - } - if (best_i < 0 || best_rank == INT_MAX) - break; - - char *merged = str_concat2(syms[best_i], syms[best_i + 1]); - if (!merged) { - for (int i = 0; i < n_syms; i++) - free(syms[i]); - free(syms); - return -1; - } - free(syms[best_i]); - free(syms[best_i + 1]); - syms[best_i] = merged; - for (int j = best_i + 1; j < n_syms - 1; j++) - syms[j] = syms[j + 1]; - n_syms--; - } - - int *ids = NULL; - int n_ids = 0, cap = 0; - for (int i = 0; i < n_syms; i++) { - int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]); - if (id < 0) { - /* Should not happen with valid vocab + merges + byte-level mapping. */ - for (int k = 0; k < n_syms; k++) - free(syms[k]); - free(syms); - free(ids); - return -1; - } - if (append_id(&ids, &n_ids, &cap, id) != 0) { - for (int k = 0; k < n_syms; k++) - free(syms[k]); - free(syms); - free(ids); - return -1; - } - } - - for (int i = 0; i < n_syms; i++) - free(syms[i]); - free(syms); - - *out_ids = ids; - *out_n = n_ids; - return 0; -} - -typedef struct { - size_t off; -} bpe_sym_ref_t; - -struct embed_tokenizer_workspace { - char *normalized; - size_t normalized_cap; - char *mapped; - size_t mapped_cap; - char *arena; - size_t arena_cap; - size_t arena_len; - bpe_sym_ref_t *syms; - int syms_cap; - int *ids; - int ids_cap; - /* Integer-pair BPE merge scratch: token id, doubly-linked live order, - * and the cached (rank, merged id) of the pair starting at each live - * position. */ - int *bpe_ids; - int *bpe_prv; - int *bpe_nxt; - int *bpe_rnk; - int *bpe_mrg; - int bpe_cap; -}; - -static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) { - if (ws->bpe_cap >= need) - return 0; - int new_cap = ws->bpe_cap ? ws->bpe_cap : 64; - while (new_cap < need) { - if (new_cap > INT_MAX / 2) - return -1; - new_cap *= 2; - } - int *p; -#define R(field) \ - p = (int *)realloc(ws->field, (size_t)new_cap * sizeof(int)); \ - if (!p) \ - return -1; \ - ws->field = p; - R(bpe_ids) - R(bpe_prv) - R(bpe_nxt) - R(bpe_rnk) - R(bpe_mrg) -#undef R - ws->bpe_cap = new_cap; - return 0; -} - -static int reserve_bytes(char **buf, size_t *cap, size_t need) { - if (*cap >= need) - return 0; - size_t new_cap = *cap ? *cap : 256; - while (new_cap < need) { - if (new_cap > SIZE_MAX / 2) - return -1; - new_cap *= 2; - } - char *p = (char *)realloc(*buf, new_cap); - if (!p) - return -1; - *buf = p; - *cap = new_cap; - return 0; -} - -static int reserve_ints(int **buf, int *cap, int need) { - if (*cap >= need) - return 0; - int new_cap = *cap ? *cap : 64; - while (new_cap < need) { - if (new_cap > INT_MAX / 2) - return -1; - new_cap *= 2; - } - int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int)); - if (!p) - return -1; - *buf = p; - *cap = new_cap; - return 0; -} - -static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) { - if (ws->syms_cap >= need) - return 0; - int new_cap = ws->syms_cap ? ws->syms_cap : 64; - while (new_cap < need) { - if (new_cap > INT_MAX / 2) - return -1; - new_cap *= 2; - } - bpe_sym_ref_t *p = (bpe_sym_ref_t *)realloc(ws->syms, (size_t)new_cap * sizeof(*ws->syms)); - if (!p) - return -1; - ws->syms = p; - ws->syms_cap = new_cap; - return 0; -} - -static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) { - if (ws->arena_len > SIZE_MAX - len - 1) - return -1; - size_t need = ws->arena_len + len + 1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0) - return -1; - *off = ws->arena_len; - memcpy(ws->arena + ws->arena_len, s, len); - ws->arena_len += len; - ws->arena[ws->arena_len++] = '\0'; - return 0; -} - -static const char *arena_str(const embed_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) { - return ws->arena + sym.off; -} - -static int -normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, const char **out) { - const unsigned char *p = (const unsigned char *)text; - const unsigned char *end = p + strlen(text); - size_t in_len = (size_t)(end - p); - if (in_len > (SIZE_MAX - 1) / 2) - return -1; - if (reserve_bytes(&ws->normalized, &ws->normalized_cap, in_len * 2 + 1) != 0) - return -1; - - size_t w = 0; - while (p < end) { - const unsigned char *cur = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) - break; - p += len; - - if (p < end) { - int mark = 0, mark_len = 0; - utf8_decode_cp_len(p, end, &mark, &mark_len); - int composed = compose_latin_mark(cp, mark); - if (composed) { - char tmp[4]; - int n = utf8_encode_cp(composed, tmp); - memcpy(ws->normalized + w, tmp, (size_t)n); - w += (size_t)n; - p += mark_len; - continue; - } - } - - memcpy(ws->normalized + w, cur, (size_t)len); - w += (size_t)len; - } - - ws->normalized[w] = '\0'; - *out = ws->normalized; - return 0; -} - -static int text_to_bpe_unicode_into(embed_tokenizer_workspace_t *ws, - const char *text, - size_t len, - const char **out) { - init_gpt2_mapping(); - if (len > (SIZE_MAX - 1) / 3) - return -1; - if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0) - return -1; - - size_t w = 0; - for (size_t i = 0; i < len; i++) { - int cp = gpt2_byte_to_unicode[(unsigned char)text[i]]; - char tmp[4]; - int n = utf8_encode_cp(cp, tmp); - memcpy(ws->mapped + w, tmp, (size_t)n); - w += (size_t)n; - } - ws->mapped[w] = '\0'; - *out = ws->mapped; - return 0; -} - -static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow, int id) { - if (*n_ids < out_cap && out_ids) - out_ids[*n_ids] = id; - else - *overflow = 1; - (*n_ids)++; - return 0; -} - -static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *mapped, - int *out_ids, - int out_cap, - int *n_ids, - int *overflow) { - if (!mapped || !*mapped) - return 0; - - int n_syms = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p;) { - int l = utf8_char_len(*p); - p += l; - n_syms++; - } - if (n_syms <= 0) - return 0; - if (reserve_syms(ws, n_syms) != 0) - return -1; - - ws->arena_len = 0; - int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p;) { - int l = utf8_char_len(*p); - size_t off = 0; - if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0) - return -1; - ws->syms[i++].off = off; - p += l; - } - - while (n_syms > 1) { - int best_rank = INT_MAX; - int best_i = -1; - for (int j = 0; j < n_syms - 1; j++) { - int r = merge_rank(tok, arena_str(ws, ws->syms[j]), arena_str(ws, ws->syms[j + 1])); - if (r < best_rank) { - best_rank = r; - best_i = j; - } - } - if (best_i < 0 || best_rank == INT_MAX) - break; - - const char *a = arena_str(ws, ws->syms[best_i]); - const char *b = arena_str(ws, ws->syms[best_i + 1]); - size_t la = strlen(a), lb = strlen(b); - if (la > SIZE_MAX - lb) - return -1; - if (reserve_bytes(&ws->arena, &ws->arena_cap, ws->arena_len + la + lb + 1) != 0) - return -1; - - a = arena_str(ws, ws->syms[best_i]); - b = arena_str(ws, ws->syms[best_i + 1]); - size_t off = ws->arena_len; - memcpy(ws->arena + ws->arena_len, a, la); - ws->arena_len += la; - memcpy(ws->arena + ws->arena_len, b, lb); - ws->arena_len += lb; - ws->arena[ws->arena_len++] = '\0'; - ws->syms[best_i].off = off; - for (int j = best_i + 1; j < n_syms - 1; j++) - ws->syms[j] = ws->syms[j + 1]; - n_syms--; - } - - for (int j = 0; j < n_syms; j++) { - int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, - arena_str(ws, ws->syms[j])); - if (id < 0) - return -1; - output_append_id(out_ids, out_cap, n_ids, overflow, id); - } - return 0; -} - -/* Integer-pair BPE merge: token ids in a doubly-linked array with cached - * pair ranks; each merge updates only the two adjacent pairs. Falls back to - * the string-based loop when the int tables are unavailable. Identical - * output: initial ids are the byte-level single-char token ids and every - * merge result is the vocab id of the concatenated pair. */ -static int encode_bpe_word_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *mapped, - int *out_ids, - int out_cap, - int *n_ids, - int *overflow) { - if (!mapped || !*mapped) - return 0; - if (!tok->int_merges || !tok->cp_to_id) - return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); - - int n = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p;) { - p += utf8_char_len(*p); - n++; - } - if (n <= 0) - return 0; - if (reserve_bpe(ws, n) != 0) - return -1; - - int *ids = ws->bpe_ids, *prv = ws->bpe_prv, *nxt = ws->bpe_nxt; - int *rnk = ws->bpe_rnk, *mrg = ws->bpe_mrg; - - int i = 0; - for (const unsigned char *p = (const unsigned char *)mapped; *p;) { - int len = utf8_char_len(*p); - int cp; - if (len == 1) - cp = *p; - else if (len == 2) - cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F); - else - cp = 512; /* byte-level alphabet never exceeds two bytes */ - int id = cp < 512 ? tok->cp_to_id[cp] : -1; - if (id < 0) - return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); - ids[i] = id; - prv[i] = i - 1; - nxt[i] = i + 1 < n ? i + 1 : -1; - i++; - p += len; - } - - for (int j = 0; j < n - 1; j++) - rnk[j] = pair_merge_lookup(tok, ids[j], ids[j + 1], &mrg[j]); - if (n > 0) - rnk[n - 1] = INT_MAX; - - for (;;) { - int best = -1, best_rank = INT_MAX; - for (int j = 0; j != -1 && nxt[j] != -1; j = nxt[j]) { - if (rnk[j] < best_rank) { - best_rank = rnk[j]; - best = j; - } - } - if (best < 0) - break; - - ids[best] = mrg[best]; - int dead = nxt[best]; - nxt[best] = nxt[dead]; - if (nxt[dead] != -1) - prv[nxt[dead]] = best; - - if (prv[best] != -1) - rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]); - rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best]) - : INT_MAX; - } - - for (int j = 0; j != -1; j = nxt[j]) - output_append_id(out_ids, out_cap, n_ids, overflow, ids[j]); - return 0; -} - -static int append_encoded_piece_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *piece, - size_t piece_len, - int *out_ids, - int out_cap, - int *n_ids, - int *overflow) { - if (piece_len == 0) - return 0; - - const char *mapped = NULL; - if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0) - return -1; - return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow); -} - -static int encode_pretokenized_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n) { - const unsigned char *start = (const unsigned char *)text; - const unsigned char *p = start; - const unsigned char *end = start + strlen(text); - int n_ids = 0; - int overflow = 0; - - while (p < end) { - const unsigned char *piece = p; - int cp = 0, len = 0; - if (!utf8_decode_cp_len(p, end, &cp, &len)) - break; - - int clen = match_contraction(p, end); - if (clen > 0) { - p += clen; - } else { - const unsigned char *next = p + len; - int next_cp = 0, next_len = 0; - int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len); - - if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) - break; - p += l; - } - } else if (is_letter_cp(cp)) { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_letter_cp(c)) - break; - p += l; - } - } else if (is_number_cp(cp)) { - p += len; - } else if (is_space_cp(cp)) { - const unsigned char *q = p; - int saw_newline = 0; - while (q < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q, end, &c, &l); - if (!is_space_cp(c)) - break; - if (is_newline_cp(c)) - saw_newline = 1; - q += l; - if (saw_newline) { - while (q < end) { - int c2 = 0, l2 = 0; - utf8_decode_cp_len(q, end, &c2, &l2); - if (!is_newline_cp(c2)) - break; - q += l2; - } - break; - } - } - - if (saw_newline) { - p = q; - } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) { - p = next + next_len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) - break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) - break; - p += l; - } - } else { - const unsigned char *last = p; - const unsigned char *q2 = p; - int count = 0; - while (q2 < end) { - int c = 0, l = 0; - utf8_decode_cp_len(q2, end, &c, &l); - if (!is_space_cp(c)) - break; - last = q2; - q2 += l; - count++; - } - if (q2 < end && count > 1) - p = last; - else - p = q2; - } - } else { - p += len; - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (is_space_cp(c) || is_word_cp(c)) - break; - p += l; - } - while (p < end) { - int c = 0, l = 0; - utf8_decode_cp_len(p, end, &c, &l); - if (!is_newline_cp(c)) - break; - p += l; - } - } - } - - if (append_encoded_piece_into(tok, ws, (const char *)piece, (size_t)(p - piece), out_ids, - out_cap, &n_ids, &overflow) != 0) - return -1; - } - - *out_n = n_ids; - return overflow ? -2 : 0; -} - -static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) { - const char *slash = strrchr(vocab_path, '/'); - if (!slash) { - if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap) - return -1; - return 0; - } - size_t dir_len = (size_t)(slash - vocab_path); - if (dir_len + 12 >= out_cap) - return -1; - memcpy(out_path, vocab_path, dir_len); - out_path[dir_len] = '\0'; - snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt"); - return 0; -} - -static void trim_newline(char *line) { - size_t n = strlen(line); - while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r')) - line[--n] = '\0'; -} - -static int parse_merge_pair(char *line, char **a, char **b) { - *a = NULL; - *b = NULL; - trim_newline(line); - if (line[0] == '\0') - return -1; - if (line[0] == '#') - return -1; - - char *sp = strchr(line, ' '); - if (!sp) - return -1; - *sp = '\0'; - char *p2 = sp + 1; - while (*p2 == ' ') - p2++; - if (*line == '\0' || *p2 == '\0') - return -1; - *a = line; - *b = p2; - return 0; -} - -static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) { - FILE *f = fopen(merges_path, "rb"); - if (!f) - return -1; - - int n_pairs = 0; - char line[8192]; - while (fgets(line, sizeof(line), f)) { - char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) == 0) - n_pairs++; - } - if (n_pairs <= 0) { - fclose(f); - return -1; - } - - tok->merge_map_cap = next_pow2(n_pairs * 2); - tok->merge_map = calloc((size_t)tok->merge_map_cap, sizeof(str_int_entry_t)); - if (!tok->merge_map) { - fclose(f); - tok->merge_map_cap = 0; - return -1; - } - - if (fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - tok->int_merges_cap = next_pow2(n_pairs * 2); - pair_merge_entry_t *pm = - (pair_merge_entry_t *)calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t)); - if (pm) { - for (int i = 0; i < tok->int_merges_cap; i++) { - pm[i].key = UINT64_MAX; - pm[i].rank = INT_MAX; - pm[i].merged_id = -1; - } - } - tok->int_merges = pm; - - int rank = 0; - while (fgets(line, sizeof(line), f)) { - char *a = NULL, *b = NULL; - if (parse_merge_pair(line, &a, &b) != 0) - continue; - - size_t la = strlen(a), lb = strlen(b); - char *key = (char *)malloc(la + 1 + lb + 1); - if (!key) - continue; - memcpy(key, a, la); - key[la] = ' '; - memcpy(key + la + 1, b, lb); - key[la + 1 + lb] = '\0'; - - if (tok->int_merges) { - /* Resolve (a, b) and the merged token a+b to vocab ids; if any - * merge is unresolvable, drop the whole int table and keep the - * string-based loop as the only path. */ - const str_int_entry_t *vm = (const str_int_entry_t *)tok->vocab_map; - int left = map_get(vm, tok->vocab_map_cap, a); - int right = map_get(vm, tok->vocab_map_cap, b); - char ab[8192]; - int merged = -1; - if (la + lb < sizeof(ab)) { - memcpy(ab, a, la); - memcpy(ab + la, b, lb); - ab[la + lb] = '\0'; - merged = map_get(vm, tok->vocab_map_cap, ab); - } - if (left < 0 || right < 0 || merged < 0) { - free(tok->int_merges); - tok->int_merges = NULL; - tok->int_merges_cap = 0; - } else { - pair_map_insert((pair_merge_entry_t *)tok->int_merges, tok->int_merges_cap, - ((uint64_t)(uint32_t)left << 32) | (uint32_t)right, rank, merged); - } - } - (void)map_insert_owned((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, &key, rank); - free(key); - rank++; - } - fclose(f); - - /* Byte-level unicode codepoint -> single-char token id table. */ - if (tok->int_merges) { - init_gpt2_mapping(); - tok->cp_to_id = (int *)malloc(512 * sizeof(int)); - if (tok->cp_to_id) { - for (int i = 0; i < 512; i++) - tok->cp_to_id[i] = -1; - for (int byte = 0; byte < 256 && tok->cp_to_id; byte++) { - int cp = gpt2_byte_to_unicode[byte]; - char buf[4]; - int n; - if (cp < 0x80) { - buf[0] = (char)cp; - n = 1; - } else { - buf[0] = (char)(0xC0 | (cp >> 6)); - buf[1] = (char)(0x80 | (cp & 0x3F)); - n = 2; - } - buf[n] = '\0'; - int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, buf); - if (id < 0) { - free(tok->cp_to_id); - tok->cp_to_id = NULL; - } else { - tok->cp_to_id[cp] = id; - } - } - } - } - return 0; -} - -/* ======================================================================== - * Public API - * ======================================================================== */ - -embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) { - FILE *f = fopen(vocab_json_path, "rb"); - if (!f) { - fprintf(stderr, "embed_tokenizer_load: cannot open %s\n", vocab_json_path); - return NULL; - } - - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return NULL; - } - long file_size = ftell(f); - if (file_size < 0 || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return NULL; - } - - char *json = (char *)malloc((size_t)file_size + 1); - if (!json || fread(json, 1, (size_t)file_size, f) != (size_t)file_size) { - fclose(f); - free(json); - return NULL; - } - fclose(f); - json[file_size] = '\0'; - - int max_id = 0; - const char *p = json; - skip_ws(&p); - if (*p != '{') { - free(json); - return NULL; - } - p++; - - const char *saved = p; - while (*p && *p != '}') { - skip_ws(&p); - if (*p == ',') { - p++; - continue; - } - if (*p == '}') - break; - - char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) - break; - skip_ws(&p); - if (*p != ':') - break; - p++; - int64_t id = parse_json_int(&p); - if (id > max_id) - max_id = (int)id; - } - - int vocab_size = max_id + 1; - embed_tokenizer_t *tok = (embed_tokenizer_t *)calloc(1, sizeof(embed_tokenizer_t)); - if (!tok) { - free(json); - return NULL; - } - tok->vocab_size = vocab_size; - tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *)); - tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *)); - if (!tok->id_to_text || !tok->id_to_bpe) { - embed_tokenizer_free(tok); - free(json); - return NULL; - } - - p = saved; - while (*p && *p != '}') { - skip_ws(&p); - if (*p == ',') { - p++; - continue; - } - if (*p == '}') - break; - - char key[4096] = {0}; - if (parse_json_string(&p, key, sizeof(key)) != 0) - break; - skip_ws(&p); - if (*p != ':') - break; - p++; - int64_t id = parse_json_int(&p); - - if (id >= 0 && id < vocab_size) { - free(tok->id_to_bpe[id]); - free(tok->id_to_text[id]); - tok->id_to_bpe[id] = strdup(key); - tok->id_to_text[id] = decode_gpt2_token(key); - } - } - free(json); - - int n_vocab_entries = 0; - for (int i = 0; i < vocab_size; i++) - if (tok->id_to_bpe[i]) - n_vocab_entries++; - tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1); - tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t)); - if (!tok->vocab_map) { - embed_tokenizer_free(tok); - return NULL; - } - for (int i = 0; i < vocab_size; i++) { - if (tok->id_to_bpe[i]) { - map_insert((str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, tok->id_to_bpe[i], i); - } - } - - char merges_path[1024]; - if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) { - if (load_merges_map(tok, merges_path) != 0 && embed_verbose >= 2) { - fprintf(stderr, - "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n", - merges_path); - } - } - - return tok; -} - -const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id) { - if (!tok || token_id < 0 || token_id >= tok->vocab_size) - return ""; - return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : ""; -} - -int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token) { - if (!tok || !token || !tok->vocab_map) - return -1; - return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token); -} - -int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !text || text[0] == '\0') - return NULL; - - char *normalized = normalize_nfc_latin(text); - if (!normalized) - return NULL; - - int *ids = NULL; - int n_ids = 0; - if (encode_pretokenized(tok, normalized, &ids, &n_ids) != 0) { - free(normalized); - free(ids); - return NULL; - } - free(normalized); - - if (out_n_tokens) - *out_n_tokens = n_ids; - return ids; -} - -embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void) { - return (embed_tokenizer_workspace_t *)calloc(1, sizeof(embed_tokenizer_workspace_t)); -} - -void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) { - if (!ws) - return; - free(ws->normalized); - free(ws->mapped); - free(ws->arena); - free(ws->syms); - free(ws->ids); - free(ws->bpe_ids); - free(ws->bpe_prv); - free(ws->bpe_nxt); - free(ws->bpe_rnk); - free(ws->bpe_mrg); - free(ws); -} - -int embed_tokenizer_encode_into(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text || out_cap < 0) - return -1; - if (text[0] == '\0') - return 0; - - const char *normalized = NULL; - if (normalize_nfc_latin_into(ws, text, &normalized) != 0) - return -1; - - int n_ids = 0; - int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, &n_ids); - if (out_n_tokens) - *out_n_tokens = n_ids; - return rc; -} - -int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok, - embed_tokenizer_workspace_t *ws, - const char *text, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text || text[0] == '\0') - return NULL; - - size_t len = strlen(text); - size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8; - if (cap_sz > (size_t)INT_MAX) - return NULL; - int cap = (int)cap_sz; - - if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0) - return NULL; - - int n_ids = 0; - int rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); - if (rc == -2) { - if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0) - return NULL; - rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids); - } - if (rc != 0 || n_ids <= 0) - return NULL; - - int *ids = (int *)malloc((size_t)n_ids * sizeof(int)); - if (!ids) - return NULL; - memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int)); - if (out_n_tokens) - *out_n_tokens = n_ids; - return ids; -} - -void embed_tokenizer_free(embed_tokenizer_t *tok) { - if (!tok) - return; - - if (tok->id_to_text) { - for (int i = 0; i < tok->vocab_size; i++) - free(tok->id_to_text[i]); - free(tok->id_to_text); - } - if (tok->id_to_bpe) { - for (int i = 0; i < tok->vocab_size; i++) - free(tok->id_to_bpe[i]); - free(tok->id_to_bpe); - } - - if (tok->merge_map) { - str_int_entry_t *m = (str_int_entry_t *)tok->merge_map; - for (int i = 0; i < tok->merge_map_cap; i++) - free(m[i].key); - free(tok->merge_map); - } - free(tok->vocab_map); - free(tok->int_merges); - free(tok->cp_to_id); - free(tok); -} |
