aboutsummaryrefslogtreecommitdiff
path: root/libffwd/tokenize.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
commitcf7d8169ec12cb3adf90d4ff78b3257e74985302 (patch)
treef666589acfd68e76bbde4e74eead44ae7819d0bb /libffwd/tokenize.c
parent91871be6ece9317c5d49353f6474583e7e28168d (diff)
downloadlangdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.tar.gz
langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.zip
Remove unused model families and inference paths
Diffstat (limited to 'libffwd/tokenize.c')
-rw-r--r--libffwd/tokenize.c324
1 files changed, 30 insertions, 294 deletions
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
index 2af151a..8e1000f 100644
--- a/libffwd/tokenize.c
+++ b/libffwd/tokenize.c
@@ -1,52 +1,16 @@
-/*
- * Model-aware tokenization: text -> token ids the model expects.
- *
- * Backend-independent, so this uses its own handle, ffwd_tok_t, instead of the
- * per-backend ffwd_t.
- *
- * ffwd_tok_open selects the tokenizer from the files present and resolves the
- * model's special-token ids.
- *
- * ffwd_tokenize applies the model's token layout: BERT/XLM-R [CLS]..[SEP], or
- * Qwen appended terminal token.
- */
-
#include "ffwd.h"
-
-#include "bpe.h"
#include "sentencepiece.h"
-#include "wordpiece.h"
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <unistd.h>
-
-typedef enum { TOK_KIND_BPE, TOK_KIND_WP, TOK_KIND_SPM } tok_kind_t;
struct ffwd_tok {
- tok_kind_t kind;
- /* exactly one trio is populated, by kind */
- tok_bpe_t *bpe;
- tok_bpe_workspace_t *bpe_ws;
- tok_wp_t *wp;
- tok_wp_workspace_t *wp_ws;
tok_spm_t *spm;
- tok_spm_workspace_t *spm_ws;
-
- int cls_id; /* [CLS] / <s> — wp/spm wrap with these */
- int sep_id; /* [SEP] / </s> */
- int context_separator_id;
- int terminal_id; /* appended on the bpe path when append_terminal */
- int append_terminal; /* Qwen3-Embedding pools the last token; see note */
-
- int is_late;
- int late_mask_id;
- int late_query_prefix_id;
- int late_document_prefix_id;
- int late_skip_ids[64];
- int n_late_skip_ids;
+ tok_spm_workspace_t *workspace;
+ int cls_id;
+ int sep_id;
};
static void set_err(char *err, size_t errlen, const char *msg) {
@@ -54,7 +18,7 @@ static void set_err(char *err, size_t errlen, const char *msg) {
snprintf(err, errlen, "%s", msg);
}
-ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen) {
+ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) {
if (!model_dir) {
set_err(err, errlen, "model_dir is NULL");
return NULL;
@@ -64,109 +28,21 @@ ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t
set_err(err, errlen, "out of memory");
return NULL;
}
- t->cls_id = -1;
- t->sep_id = -1;
- t->terminal_id = -1;
- t->is_late = is_late;
-
- char path[1024];
-
- /* WordPiece (BERT family): vocab.txt. */
- snprintf(path, sizeof(path), "%s/vocab.txt", model_dir);
- if (access(path, R_OK) == 0) {
- t->kind = TOK_KIND_WP;
- t->wp = tok_wp_load(model_dir);
- if (!t->wp) {
- set_err(err, errlen, "failed to load WordPiece tokenizer");
- goto fail;
- }
- t->wp_ws = tok_wp_workspace_new();
- if (!t->wp_ws) {
- set_err(err, errlen, "out of memory");
- goto fail;
- }
- t->cls_id = tok_wp_token_id(t->wp, "[CLS]");
- t->sep_id = tok_wp_token_id(t->wp, "[SEP]");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "WordPiece vocab missing [CLS]/[SEP]");
- goto fail;
- }
- return t;
- }
-
- /* SentencePiece (XLM-R family): sentencepiece.bpe.model / tokenizer.model /
- * spiece.model. */
- snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir);
- if (access(path, R_OK) != 0) {
- snprintf(path, sizeof(path), "%s/tokenizer.model", model_dir);
- if (access(path, R_OK) != 0)
- snprintf(path, sizeof(path), "%s/spiece.model", model_dir);
- }
- if (access(path, R_OK) == 0) {
- t->kind = TOK_KIND_SPM;
- t->spm = tok_spm_load(model_dir);
- if (!t->spm) {
- set_err(err, errlen, "failed to load SentencePiece tokenizer");
- goto fail;
- }
- t->spm_ws = tok_spm_workspace_new();
- if (!t->spm_ws) {
- set_err(err, errlen, "out of memory");
- goto fail;
- }
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
- return t;
- }
-
- /* Byte-level BPE (Qwen family): vocab.json. */
- snprintf(path, sizeof(path), "%s/vocab.json", model_dir);
- t->kind = TOK_KIND_BPE;
- t->bpe = tok_bpe_load(path);
- if (!t->bpe) {
- set_err(err, errlen, "failed to load tokenizer");
+ t->spm = tok_spm_load(model_dir);
+ if (!t->spm) {
+ set_err(err, errlen, "failed to load SentencePiece tokenizer");
goto fail;
}
- t->bpe_ws = tok_bpe_workspace_new();
- if (!t->bpe_ws) {
+ t->workspace = tok_spm_workspace_new();
+ if (!t->workspace) {
set_err(err, errlen, "out of memory");
goto fail;
}
- /* The contextual chunk separator is the tokenizer's <|endoftext|>; released
- * models keep it out of vocab.json, so the family constant is the fallback.
- * Qwen3-Embedding pools the last token (the same <|endoftext|>), so the bpe
- * path appends it. A config that disables append_terminal_token would need
- * to override this; the released models all enable it. */
- int sep = tok_bpe_token_id(t->bpe, "<|endoftext|>");
- t->context_separator_id = sep >= 0 ? sep : FFWD_CONTEXT_SEPARATOR_TOKEN_ID;
- t->terminal_id = t->context_separator_id;
- /* Off until the caller sets it from config->append_terminal_token; it is
- * model-specific (pplx-embed does not append, for example). */
- t->append_terminal = 0;
-
- if (is_late) {
- int id = tok_bpe_token_id(t->bpe, "[MASK]");
- t->late_mask_id = id >= 0 ? id : FFWD_LATE_MASK_TOKEN_ID;
- id = tok_bpe_token_id(t->bpe, "[Q]");
- t->late_query_prefix_id = id >= 0 ? id : FFWD_LATE_QUERY_PREFIX_ID;
- id = tok_bpe_token_id(t->bpe, "[D]");
- t->late_document_prefix_id = id >= 0 ? id : FFWD_LATE_DOCUMENT_PREFIX_ID;
-
- const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
- for (const char *p = punct;
- *p && t->n_late_skip_ids < (int)(sizeof(t->late_skip_ids) / sizeof(t->late_skip_ids[0]));
- p++) {
- char text[2] = {*p, '\0'};
- int n_ids = 0;
- int *ids = tok_bpe_encode(t->bpe, text, &n_ids);
- if (ids && n_ids == 1)
- t->late_skip_ids[t->n_late_skip_ids++] = ids[0];
- free(ids);
- }
+ t->cls_id = tok_spm_token_id(t->spm, "<s>");
+ t->sep_id = tok_spm_token_id(t->spm, "</s>");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
+ goto fail;
}
return t;
@@ -178,172 +54,32 @@ fail:
void ffwd_tok_free(ffwd_tok_t *t) {
if (!t)
return;
- if (t->bpe_ws)
- tok_bpe_workspace_free(t->bpe_ws);
- if (t->bpe)
- tok_bpe_free(t->bpe);
- if (t->wp_ws)
- tok_wp_workspace_free(t->wp_ws);
- if (t->wp)
- tok_wp_free(t->wp);
- if (t->spm_ws)
- tok_spm_workspace_free(t->spm_ws);
- if (t->spm)
- tok_spm_free(t->spm);
+ tok_spm_workspace_free(t->workspace);
+ tok_spm_free(t->spm);
free(t);
}
-int ffwd_tok_context_separator_id(const ffwd_tok_t *t) { return t ? t->context_separator_id : -1; }
-
-void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on) {
- if (t)
- t->append_terminal = on ? 1 : 0;
-}
-
-/* Encode text and apply the model's special-token layout. Returns malloc'd ids
- * and sets *n_out; NULL on error. */
-static int *tokenize_core(ffwd_tok_t *t, const char *text, int *n_out) {
- *n_out = 0;
-
- if (t->kind == TOK_KIND_WP || t->kind == TOK_KIND_SPM) {
- int n = 0;
- int *core = (t->kind == TOK_KIND_WP)
- ? tok_wp_encode_with_workspace(t->wp, t->wp_ws, text, &n)
- : tok_spm_encode_with_workspace(t->spm, t->spm_ws, text, &n);
- /* Wrap [CLS] core [SEP]. A whitespace-only input cleans to zero core
- * tokens (core == NULL, n == 0); the bare [CLS][SEP] pair is what
- * Hugging Face produces for it too. Grow in place so only the one
- * allocation is touched. */
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- return NULL;
- }
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return NULL;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
- *n_out = n + 2;
- return ids;
- }
-
- /* BPE. */
- int n = 0;
- int *ids = tok_bpe_encode_with_workspace(t->bpe, t->bpe_ws, text, &n);
- if (!ids || n <= 0) {
- free(ids);
- return NULL;
- }
- if (t->append_terminal) {
- if (n == INT_MAX) {
- free(ids);
- return NULL;
- }
- int *grown = realloc(ids, (size_t)(n + 1) * sizeof(*ids));
- if (!grown) {
- free(ids);
- return NULL;
- }
- ids = grown;
- ids[n++] = t->terminal_id;
- }
- *n_out = n;
- return ids;
-}
-
-int *ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out) {
+int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) {
if (n_out)
*n_out = 0;
if (!t || !text)
return NULL;
-
int n = 0;
- int *ids;
- if (!query_instruct) {
- ids = tokenize_core(t, text, &n);
- } else {
- /* Tokenize the instruction and the input in one pass. */
- size_t plen = strlen(query_instruct);
- size_t tlen = strlen(text);
- char *buf = malloc(plen + tlen + 1);
- if (!buf)
- return NULL;
- memcpy(buf, query_instruct, plen);
- memcpy(buf + plen, text, tlen + 1);
- ids = tokenize_core(t, buf, &n);
- free(buf);
+ int *core = tok_spm_encode_with_workspace(t->spm, t->workspace, text, &n);
+ if (n < 0 || n > INT_MAX - 2 || (!core && n > 0)) {
+ free(core);
+ return NULL;
}
- if (!ids)
+ /* Empty text still produces the two special tokens. */
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
return NULL;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = t->cls_id;
+ ids[n + 1] = t->sep_id;
if (n_out)
- *n_out = n;
+ *n_out = n + 2;
return ids;
}
-
-static int late_id_is_skipped(const ffwd_tok_t *t, int id) {
- for (int i = 0; i < t->n_late_skip_ids; i++) {
- if (t->late_skip_ids[i] == id)
- return 1;
- }
- return 0;
-}
-
-int ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out) {
- memset(out, 0, sizeof(*out));
- if (!t || !text)
- return -1;
-
- int raw_n = 0;
- int *raw = tokenize_core(t, text, &raw_n);
- if (!raw)
- return -1;
-
- int target = raw_n + 1;
- if (is_query && target < FFWD_LATE_QUERY_TOKENS)
- target = FFWD_LATE_QUERY_TOKENS;
-
- out->ids = malloc((size_t)target * sizeof(*out->ids));
- if (!out->ids) {
- free(raw);
- return -1;
- }
- out->ids[0] = raw[0];
- out->ids[1] = is_query ? t->late_query_prefix_id : t->late_document_prefix_id;
- if (raw_n > 1)
- memcpy(out->ids + 2, raw + 1, (size_t)(raw_n - 1) * sizeof(*out->ids));
- out->n_tokens = raw_n + 1;
- free(raw);
-
- if (is_query) {
- while (out->n_tokens < FFWD_LATE_QUERY_TOKENS)
- out->ids[out->n_tokens++] = t->late_mask_id;
- out->n_keep = out->n_tokens;
- } else {
- out->keep = malloc((size_t)out->n_tokens * sizeof(*out->keep));
- if (!out->keep) {
- ffwd_late_tokens_free(out);
- return -1;
- }
- for (int i = 0; i < out->n_tokens; i++) {
- if (!late_id_is_skipped(t, out->ids[i]))
- out->keep[out->n_keep++] = i;
- }
- }
-
- if (out->n_keep <= 0) {
- ffwd_late_tokens_free(out);
- return -1;
- }
- return 0;
-}
-
-void ffwd_late_tokens_free(ffwd_late_tokens_t *out) {
- if (!out)
- return;
- free(out->ids);
- free(out->keep);
- memset(out, 0, sizeof(*out));
-}