diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 16:27:47 +0200 |
| commit | d1ba1ebc06ef0d62ca932388fae83e6992594719 (patch) | |
| tree | f962207b779016593258f990348216999a58766a /src/tokenizer_wordpiece.h | |
| parent | be20dfda0f836808dccfc14db244592e41ba28bc (diff) | |
| download | ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.tar.gz ffwd-d1ba1ebc06ef0d62ca932388fae83e6992594719.zip | |
chore: move sources to src/ with proper names, drop qwen_ prefix
Diffstat (limited to 'src/tokenizer_wordpiece.h')
| -rw-r--r-- | src/tokenizer_wordpiece.h | 79 |
1 files changed, 79 insertions, 0 deletions
diff --git a/src/tokenizer_wordpiece.h b/src/tokenizer_wordpiece.h new file mode 100644 index 0000000..59cb2bb --- /dev/null +++ b/src/tokenizer_wordpiece.h @@ -0,0 +1,79 @@ +/* + * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer. + * + * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional + * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) + * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes + * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the + * byte-level BPE backend leaves terminal/separator tokens to the frontend. + * + * The public surface mirrors embed_tokenizer.h (load / encode with a reusable + * workspace / decode / resolve a named special token / free) so both can sit + * behind one tokenizer interface later. + */ + +#ifndef WORDPIECE_TOKENIZER_H +#define WORDPIECE_TOKENIZER_H + +#ifndef EMBED_API +#if defined(__GNUC__) +#define EMBED_API __attribute__((visibility("default"))) +#else +#define EMBED_API +#endif +#endif + +typedef struct { + char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */ + int vocab_size; + int unk_id; /* [UNK] id, resolved at load */ + int do_lower_case; /* lowercase + strip accents before WordPiece */ + int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */ + + /* Opaque token-string -> id hash map. */ + void *vocab_map; + int vocab_map_cap; +} wordpiece_tokenizer_t; + +typedef struct wordpiece_workspace wordpiece_workspace_t; + +/* Load from <model_dir>/vocab.txt, reading do_lower_case from + * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on + * error (missing vocab.txt, missing [UNK], allocation failure). */ +EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir); + +/* Decode a single id to its vocab token string (internal pointer), or NULL. */ +EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id); + +/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ +EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token); + +/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on + * error (and *out_n_tokens = 0). */ +EMBED_API int * +wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens); + +/* Reusable scratch for allocation-light repeated encoding. */ +EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void); +EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws); + +/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 + * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to + * the number of tokens needed/emitted in all cases. */ +EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok, + wordpiece_workspace_t *ws, + const char *text, + int *out_ids, + int out_cap, + int *out_n_tokens); + +/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and + * performing only the final returned int[] allocation. */ +EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok, + wordpiece_workspace_t *ws, + const char *text, + int *out_n_tokens); + +EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok); + +#endif /* WORDPIECE_TOKENIZER_H */ |
