aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-05-24 00:47:43 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:18:21 +0200
commit0ff6778eac127c82afc53d03614954ccab35ebdb (patch)
tree1a7ab6799ff3fbfa32a59c5e9136078c8c85b9b6
parent49663acfb43a08b7c21e5414fe0ca2a80027deec (diff)
downloadffwd-0ff6778eac127c82afc53d03614954ccab35ebdb.tar.gz
ffwd-0ff6778eac127c82afc53d03614954ccab35ebdb.zip
tokenizer: reduce merge allocation
-rw-r--r--qwen_asr_tokenizer.c51
1 files changed, 41 insertions, 10 deletions
diff --git a/qwen_asr_tokenizer.c b/qwen_asr_tokenizer.c
index 43fb504..8c1b84a 100644
--- a/qwen_asr_tokenizer.c
+++ b/qwen_asr_tokenizer.c
@@ -205,6 +205,32 @@ static uint64_t fnv1a_hash(const char *s) {
return h;
}
+static uint64_t fnv1a_hash_pair(const char *a, const char *b) {
+ uint64_t h = 1469598103934665603ULL;
+ while (*a) {
+ h ^= (unsigned char)*a++;
+ h *= 1099511628211ULL;
+ }
+ h ^= (unsigned char)' ';
+ h *= 1099511628211ULL;
+ while (*b) {
+ h ^= (unsigned char)*b++;
+ h *= 1099511628211ULL;
+ }
+ return h;
+}
+
+static int pair_key_eq(const char *key, const char *a, const char *b) {
+ while (*a) {
+ if (*key++ != *a++) return 0;
+ }
+ if (*key++ != ' ') return 0;
+ while (*b) {
+ if (*key++ != *b++) return 0;
+ }
+ return *key == '\0';
+}
+
static int next_pow2(int x) {
int p = 1;
while (p < x) p <<= 1;
@@ -242,6 +268,19 @@ static int map_get(const str_int_entry_t *map, int cap, const char *key) {
return -1;
}
+static int map_get_pair(const str_int_entry_t *map, int cap,
+ const char *a, const char *b) {
+ if (!map || cap <= 0 || !a || !b) return -1;
+ int mask = cap - 1;
+ int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask);
+ for (int i = 0; i < cap; i++) {
+ int idx = (pos + i) & mask;
+ if (!map[idx].key) return -1;
+ if (pair_key_eq(map[idx].key, a, b)) return map[idx].value;
+ }
+ return -1;
+}
+
/* ========================================================================
* BPE helpers
* ======================================================================== */
@@ -303,16 +342,8 @@ static char *str_concat2(const char *a, const char *b) {
static int merge_rank(const qwen_tokenizer_t *tok, const char *a, const char *b) {
if (!tok->merge_map || tok->merge_map_cap <= 0) return INT_MAX;
- size_t la = strlen(a), lb = strlen(b);
- char *pair = (char *)malloc(la + 1 + lb + 1);
- if (!pair) return INT_MAX;
- memcpy(pair, a, la);
- pair[la] = ' ';
- memcpy(pair + la + 1, b, lb);
- pair[la + 1 + lb] = '\0';
-
- int rank = map_get((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, pair);
- free(pair);
+ int rank = map_get_pair((const str_int_entry_t *)tok->merge_map,
+ tok->merge_map_cap, a, b);
return rank >= 0 ? rank : INT_MAX;
}