diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:10:56 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 05:57:19 +0200 |
| commit | 6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch) | |
| tree | c23afb50c28b9deaddae5c6526a74ee11cf2d865 /src/tokenizer_wordpiece.h | |
| parent | d4bdb8dae753462f621507d5fc3582e0e53da35c (diff) | |
| download | ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip | |
all: split Makefile and reorg library/tools
Diffstat (limited to 'src/tokenizer_wordpiece.h')
| -rw-r--r-- | src/tokenizer_wordpiece.h | 79 |
1 files changed, 0 insertions, 79 deletions
diff --git a/src/tokenizer_wordpiece.h b/src/tokenizer_wordpiece.h deleted file mode 100644 index 59cb2bb..0000000 --- a/src/tokenizer_wordpiece.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer. - * - * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional - * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) - * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes - * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the - * byte-level BPE backend leaves terminal/separator tokens to the frontend. - * - * The public surface mirrors embed_tokenizer.h (load / encode with a reusable - * workspace / decode / resolve a named special token / free) so both can sit - * behind one tokenizer interface later. - */ - -#ifndef WORDPIECE_TOKENIZER_H -#define WORDPIECE_TOKENIZER_H - -#ifndef EMBED_API -#if defined(__GNUC__) -#define EMBED_API __attribute__((visibility("default"))) -#else -#define EMBED_API -#endif -#endif - -typedef struct { - char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */ - int vocab_size; - int unk_id; /* [UNK] id, resolved at load */ - int do_lower_case; /* lowercase + strip accents before WordPiece */ - int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */ - - /* Opaque token-string -> id hash map. */ - void *vocab_map; - int vocab_map_cap; -} wordpiece_tokenizer_t; - -typedef struct wordpiece_workspace wordpiece_workspace_t; - -/* Load from <model_dir>/vocab.txt, reading do_lower_case from - * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on - * error (missing vocab.txt, missing [UNK], allocation failure). */ -EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir); - -/* Decode a single id to its vocab token string (internal pointer), or NULL. */ -EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id); - -/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ -EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token); - -/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on - * error (and *out_n_tokens = 0). */ -EMBED_API int * -wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens); - -/* Reusable scratch for allocation-light repeated encoding. */ -EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void); -EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws); - -/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 - * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to - * the number of tokens needed/emitted in all cases. */ -EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens); - -/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and - * performing only the final returned int[] allocation. */ -EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok, - wordpiece_workspace_t *ws, - const char *text, - int *out_n_tokens); - -EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok); - -#endif /* WORDPIECE_TOKENIZER_H */ |
