aboutsummaryrefslogtreecommitdiff
path: root/libffwd/tokenize.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:54:13 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:54:13 +0200
commit91871be6ece9317c5d49353f6474583e7e28168d (patch)
tree8cabf2546322a9e1b724bf8413f32de54378824f /libffwd/tokenize.c
parent35ab9c12ba4d396e6cba99d6ea947e26becedad4 (diff)
downloadlangdet-91871be6ece9317c5d49353f6474583e7e28168d.tar.gz
langdet-91871be6ece9317c5d49353f6474583e7e28168d.zip
Add native CPU language detection CLI
Diffstat (limited to 'libffwd/tokenize.c')
-rw-r--r--libffwd/tokenize.c349
1 files changed, 349 insertions, 0 deletions
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
new file mode 100644
index 0000000..2af151a
--- /dev/null
+++ b/libffwd/tokenize.c
@@ -0,0 +1,349 @@
+/*
+ * Model-aware tokenization: text -> token ids the model expects.
+ *
+ * Backend-independent, so this uses its own handle, ffwd_tok_t, instead of the
+ * per-backend ffwd_t.
+ *
+ * ffwd_tok_open selects the tokenizer from the files present and resolves the
+ * model's special-token ids.
+ *
+ * ffwd_tokenize applies the model's token layout: BERT/XLM-R [CLS]..[SEP], or
+ * Qwen appended terminal token.
+ */
+
+#include "ffwd.h"
+
+#include "bpe.h"
+#include "sentencepiece.h"
+#include "wordpiece.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+typedef enum { TOK_KIND_BPE, TOK_KIND_WP, TOK_KIND_SPM } tok_kind_t;
+
+struct ffwd_tok {
+ tok_kind_t kind;
+ /* exactly one trio is populated, by kind */
+ tok_bpe_t *bpe;
+ tok_bpe_workspace_t *bpe_ws;
+ tok_wp_t *wp;
+ tok_wp_workspace_t *wp_ws;
+ tok_spm_t *spm;
+ tok_spm_workspace_t *spm_ws;
+
+ int cls_id; /* [CLS] / <s> — wp/spm wrap with these */
+ int sep_id; /* [SEP] / </s> */
+ int context_separator_id;
+ int terminal_id; /* appended on the bpe path when append_terminal */
+ int append_terminal; /* Qwen3-Embedding pools the last token; see note */
+
+ int is_late;
+ int late_mask_id;
+ int late_query_prefix_id;
+ int late_document_prefix_id;
+ int late_skip_ids[64];
+ int n_late_skip_ids;
+};
+
+static void set_err(char *err, size_t errlen, const char *msg) {
+ if (err && errlen)
+ snprintf(err, errlen, "%s", msg);
+}
+
+ffwd_tok_t *ffwd_tok_open(const char *model_dir, int is_late, char *err, size_t errlen) {
+ if (!model_dir) {
+ set_err(err, errlen, "model_dir is NULL");
+ return NULL;
+ }
+ ffwd_tok_t *t = calloc(1, sizeof(*t));
+ if (!t) {
+ set_err(err, errlen, "out of memory");
+ return NULL;
+ }
+ t->cls_id = -1;
+ t->sep_id = -1;
+ t->terminal_id = -1;
+ t->is_late = is_late;
+
+ char path[1024];
+
+ /* WordPiece (BERT family): vocab.txt. */
+ snprintf(path, sizeof(path), "%s/vocab.txt", model_dir);
+ if (access(path, R_OK) == 0) {
+ t->kind = TOK_KIND_WP;
+ t->wp = tok_wp_load(model_dir);
+ if (!t->wp) {
+ set_err(err, errlen, "failed to load WordPiece tokenizer");
+ goto fail;
+ }
+ t->wp_ws = tok_wp_workspace_new();
+ if (!t->wp_ws) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ t->cls_id = tok_wp_token_id(t->wp, "[CLS]");
+ t->sep_id = tok_wp_token_id(t->wp, "[SEP]");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "WordPiece vocab missing [CLS]/[SEP]");
+ goto fail;
+ }
+ return t;
+ }
+
+ /* SentencePiece (XLM-R family): sentencepiece.bpe.model / tokenizer.model /
+ * spiece.model. */
+ snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir);
+ if (access(path, R_OK) != 0) {
+ snprintf(path, sizeof(path), "%s/tokenizer.model", model_dir);
+ if (access(path, R_OK) != 0)
+ snprintf(path, sizeof(path), "%s/spiece.model", model_dir);
+ }
+ if (access(path, R_OK) == 0) {
+ t->kind = TOK_KIND_SPM;
+ t->spm = tok_spm_load(model_dir);
+ if (!t->spm) {
+ set_err(err, errlen, "failed to load SentencePiece tokenizer");
+ goto fail;
+ }
+ t->spm_ws = tok_spm_workspace_new();
+ if (!t->spm_ws) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ t->cls_id = tok_spm_token_id(t->spm, "<s>");
+ t->sep_id = tok_spm_token_id(t->spm, "</s>");
+ if (t->cls_id < 0 || t->sep_id < 0) {
+ set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
+ goto fail;
+ }
+ return t;
+ }
+
+ /* Byte-level BPE (Qwen family): vocab.json. */
+ snprintf(path, sizeof(path), "%s/vocab.json", model_dir);
+ t->kind = TOK_KIND_BPE;
+ t->bpe = tok_bpe_load(path);
+ if (!t->bpe) {
+ set_err(err, errlen, "failed to load tokenizer");
+ goto fail;
+ }
+ t->bpe_ws = tok_bpe_workspace_new();
+ if (!t->bpe_ws) {
+ set_err(err, errlen, "out of memory");
+ goto fail;
+ }
+ /* The contextual chunk separator is the tokenizer's <|endoftext|>; released
+ * models keep it out of vocab.json, so the family constant is the fallback.
+ * Qwen3-Embedding pools the last token (the same <|endoftext|>), so the bpe
+ * path appends it. A config that disables append_terminal_token would need
+ * to override this; the released models all enable it. */
+ int sep = tok_bpe_token_id(t->bpe, "<|endoftext|>");
+ t->context_separator_id = sep >= 0 ? sep : FFWD_CONTEXT_SEPARATOR_TOKEN_ID;
+ t->terminal_id = t->context_separator_id;
+ /* Off until the caller sets it from config->append_terminal_token; it is
+ * model-specific (pplx-embed does not append, for example). */
+ t->append_terminal = 0;
+
+ if (is_late) {
+ int id = tok_bpe_token_id(t->bpe, "[MASK]");
+ t->late_mask_id = id >= 0 ? id : FFWD_LATE_MASK_TOKEN_ID;
+ id = tok_bpe_token_id(t->bpe, "[Q]");
+ t->late_query_prefix_id = id >= 0 ? id : FFWD_LATE_QUERY_PREFIX_ID;
+ id = tok_bpe_token_id(t->bpe, "[D]");
+ t->late_document_prefix_id = id >= 0 ? id : FFWD_LATE_DOCUMENT_PREFIX_ID;
+
+ const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
+ for (const char *p = punct;
+ *p && t->n_late_skip_ids < (int)(sizeof(t->late_skip_ids) / sizeof(t->late_skip_ids[0]));
+ p++) {
+ char text[2] = {*p, '\0'};
+ int n_ids = 0;
+ int *ids = tok_bpe_encode(t->bpe, text, &n_ids);
+ if (ids && n_ids == 1)
+ t->late_skip_ids[t->n_late_skip_ids++] = ids[0];
+ free(ids);
+ }
+ }
+ return t;
+
+fail:
+ ffwd_tok_free(t);
+ return NULL;
+}
+
+void ffwd_tok_free(ffwd_tok_t *t) {
+ if (!t)
+ return;
+ if (t->bpe_ws)
+ tok_bpe_workspace_free(t->bpe_ws);
+ if (t->bpe)
+ tok_bpe_free(t->bpe);
+ if (t->wp_ws)
+ tok_wp_workspace_free(t->wp_ws);
+ if (t->wp)
+ tok_wp_free(t->wp);
+ if (t->spm_ws)
+ tok_spm_workspace_free(t->spm_ws);
+ if (t->spm)
+ tok_spm_free(t->spm);
+ free(t);
+}
+
+int ffwd_tok_context_separator_id(const ffwd_tok_t *t) { return t ? t->context_separator_id : -1; }
+
+void ffwd_tok_set_append_terminal(ffwd_tok_t *t, int on) {
+ if (t)
+ t->append_terminal = on ? 1 : 0;
+}
+
+/* Encode text and apply the model's special-token layout. Returns malloc'd ids
+ * and sets *n_out; NULL on error. */
+static int *tokenize_core(ffwd_tok_t *t, const char *text, int *n_out) {
+ *n_out = 0;
+
+ if (t->kind == TOK_KIND_WP || t->kind == TOK_KIND_SPM) {
+ int n = 0;
+ int *core = (t->kind == TOK_KIND_WP)
+ ? tok_wp_encode_with_workspace(t->wp, t->wp_ws, text, &n)
+ : tok_spm_encode_with_workspace(t->spm, t->spm_ws, text, &n);
+ /* Wrap [CLS] core [SEP]. A whitespace-only input cleans to zero core
+ * tokens (core == NULL, n == 0); the bare [CLS][SEP] pair is what
+ * Hugging Face produces for it too. Grow in place so only the one
+ * allocation is touched. */
+ if (n < 0 || n > INT_MAX - 2) {
+ free(core);
+ return NULL;
+ }
+ int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
+ if (!ids) {
+ free(core);
+ return NULL;
+ }
+ memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
+ ids[0] = t->cls_id;
+ ids[n + 1] = t->sep_id;
+ *n_out = n + 2;
+ return ids;
+ }
+
+ /* BPE. */
+ int n = 0;
+ int *ids = tok_bpe_encode_with_workspace(t->bpe, t->bpe_ws, text, &n);
+ if (!ids || n <= 0) {
+ free(ids);
+ return NULL;
+ }
+ if (t->append_terminal) {
+ if (n == INT_MAX) {
+ free(ids);
+ return NULL;
+ }
+ int *grown = realloc(ids, (size_t)(n + 1) * sizeof(*ids));
+ if (!grown) {
+ free(ids);
+ return NULL;
+ }
+ ids = grown;
+ ids[n++] = t->terminal_id;
+ }
+ *n_out = n;
+ return ids;
+}
+
+int *ffwd_tokenize(ffwd_tok_t *t, const char *text, const char *query_instruct, int *n_out) {
+ if (n_out)
+ *n_out = 0;
+ if (!t || !text)
+ return NULL;
+
+ int n = 0;
+ int *ids;
+ if (!query_instruct) {
+ ids = tokenize_core(t, text, &n);
+ } else {
+ /* Tokenize the instruction and the input in one pass. */
+ size_t plen = strlen(query_instruct);
+ size_t tlen = strlen(text);
+ char *buf = malloc(plen + tlen + 1);
+ if (!buf)
+ return NULL;
+ memcpy(buf, query_instruct, plen);
+ memcpy(buf + plen, text, tlen + 1);
+ ids = tokenize_core(t, buf, &n);
+ free(buf);
+ }
+ if (!ids)
+ return NULL;
+ if (n_out)
+ *n_out = n;
+ return ids;
+}
+
+static int late_id_is_skipped(const ffwd_tok_t *t, int id) {
+ for (int i = 0; i < t->n_late_skip_ids; i++) {
+ if (t->late_skip_ids[i] == id)
+ return 1;
+ }
+ return 0;
+}
+
+int ffwd_tokenize_late(ffwd_tok_t *t, const char *text, int is_query, ffwd_late_tokens_t *out) {
+ memset(out, 0, sizeof(*out));
+ if (!t || !text)
+ return -1;
+
+ int raw_n = 0;
+ int *raw = tokenize_core(t, text, &raw_n);
+ if (!raw)
+ return -1;
+
+ int target = raw_n + 1;
+ if (is_query && target < FFWD_LATE_QUERY_TOKENS)
+ target = FFWD_LATE_QUERY_TOKENS;
+
+ out->ids = malloc((size_t)target * sizeof(*out->ids));
+ if (!out->ids) {
+ free(raw);
+ return -1;
+ }
+ out->ids[0] = raw[0];
+ out->ids[1] = is_query ? t->late_query_prefix_id : t->late_document_prefix_id;
+ if (raw_n > 1)
+ memcpy(out->ids + 2, raw + 1, (size_t)(raw_n - 1) * sizeof(*out->ids));
+ out->n_tokens = raw_n + 1;
+ free(raw);
+
+ if (is_query) {
+ while (out->n_tokens < FFWD_LATE_QUERY_TOKENS)
+ out->ids[out->n_tokens++] = t->late_mask_id;
+ out->n_keep = out->n_tokens;
+ } else {
+ out->keep = malloc((size_t)out->n_tokens * sizeof(*out->keep));
+ if (!out->keep) {
+ ffwd_late_tokens_free(out);
+ return -1;
+ }
+ for (int i = 0; i < out->n_tokens; i++) {
+ if (!late_id_is_skipped(t, out->ids[i]))
+ out->keep[out->n_keep++] = i;
+ }
+ }
+
+ if (out->n_keep <= 0) {
+ ffwd_late_tokens_free(out);
+ return -1;
+ }
+ return 0;
+}
+
+void ffwd_late_tokens_free(ffwd_late_tokens_t *out) {
+ if (!out)
+ return;
+ free(out->ids);
+ free(out->keep);
+ memset(out, 0, sizeof(*out));
+}