aboutsummaryrefslogtreecommitdiff
path: root/src/tokenizer_wordpiece.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /src/tokenizer_wordpiece.h
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'src/tokenizer_wordpiece.h')
-rw-r--r--src/tokenizer_wordpiece.h79
1 files changed, 0 insertions, 79 deletions
diff --git a/src/tokenizer_wordpiece.h b/src/tokenizer_wordpiece.h
deleted file mode 100644
index 59cb2bb..0000000
--- a/src/tokenizer_wordpiece.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer.
- *
- * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional
- * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting)
- * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes
- * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the
- * byte-level BPE backend leaves terminal/separator tokens to the frontend.
- *
- * The public surface mirrors embed_tokenizer.h (load / encode with a reusable
- * workspace / decode / resolve a named special token / free) so both can sit
- * behind one tokenizer interface later.
- */
-
-#ifndef WORDPIECE_TOKENIZER_H
-#define WORDPIECE_TOKENIZER_H
-
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef struct {
- char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */
- int vocab_size;
- int unk_id; /* [UNK] id, resolved at load */
- int do_lower_case; /* lowercase + strip accents before WordPiece */
- int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */
-
- /* Opaque token-string -> id hash map. */
- void *vocab_map;
- int vocab_map_cap;
-} wordpiece_tokenizer_t;
-
-typedef struct wordpiece_workspace wordpiece_workspace_t;
-
-/* Load from <model_dir>/vocab.txt, reading do_lower_case from
- * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on
- * error (missing vocab.txt, missing [UNK], allocation failure). */
-EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir);
-
-/* Decode a single id to its vocab token string (internal pointer), or NULL. */
-EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id);
-
-/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
-EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token);
-
-/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on
- * error (and *out_n_tokens = 0). */
-EMBED_API int *
-wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens);
-
-/* Reusable scratch for allocation-light repeated encoding. */
-EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void);
-EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws);
-
-/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1
- * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
- * the number of tokens needed/emitted in all cases. */
-EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and
- * performing only the final returned int[] allocation. */
-EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok);
-
-#endif /* WORDPIECE_TOKENIZER_H */