diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-27 00:58:05 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-27 01:03:41 +0200 |
| commit | af55803d0dff9eb692c760c8d486e736c945061d (patch) | |
| tree | 0875a1112bf98304e445e0544c4309670d6f4769 /libffwd/wordpiece.h | |
| parent | 0f861b7f3098750ddee4f6570f3be228d1da5320 (diff) | |
| download | ffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip | |
chore: clean up comment formatting and wording
Diffstat (limited to 'libffwd/wordpiece.h')
| -rw-r--r-- | libffwd/wordpiece.h | 44 |
1 files changed, 23 insertions, 21 deletions
diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h index c93140a..f8a63d6 100644 --- a/libffwd/wordpiece.h +++ b/libffwd/wordpiece.h @@ -1,15 +1,16 @@ /* - * wordpiece.h - BERT-family WordPiece tokenizer. + * BERT-family WordPiece tokenizer. * - * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional - * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting) - * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes - * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the - * byte-level BPE backend leaves terminal/separator tokens to the frontend. + * Implements BERT tokenization: BasicTokenizer (cleanup, optional lowercase + * and accent stripping, CJK spacing, whitespace splitting, and punctuation + * splitting), then greedy longest-match WordPiece using vocab.txt. * - * The public surface mirrors bpe.h (load / encode with a reusable - * workspace / decode / resolve a named special token / free) so both can sit - * behind one tokenizer interface later. + * Encodes text only. The frontend adds [CLS]/[SEP], just as the byte-level BPE + * backend leaves terminal and separator tokens to the frontend. + * + * Public API mirrors bpe.h: load, encode with reusable workspace, decode, + * resolve named special tokens, and free. This lets both tokenizers share one + * tokenizer interface later. */ #ifndef TOK_WP_H @@ -29,28 +30,29 @@ typedef struct { typedef struct tok_wp_workspace tok_wp_workspace_t; -/* Load from <model_dir>/vocab.txt, reading do_lower_case from - * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on - * error (missing vocab.txt, missing [UNK], allocation failure). */ +/* Load <model_dir>/vocab.txt and do_lower_case from + * <model_dir>/tokenizer_config.json; defaults to lowercase. + * Returns NULL on error: missing vocab.txt, missing [UNK], or allocation failure. */ tok_wp_t *tok_wp_load(const char *model_dir); -/* Decode a single id to its vocab token string (internal pointer), or NULL. */ +/* Decode one id to its vocab token string, or NULL. + * Returns an internal pointer. */ const char *tok_wp_decode(const tok_wp_t *tok, int id); /* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */ int tok_wp_token_id(const tok_wp_t *tok, const char *token); -/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on - * error (and *out_n_tokens = 0). */ +/* Encode UTF-8 text into a malloc'd id array and set *out_n_tokens. + * Returns NULL on error and sets *out_n_tokens = 0. */ int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens); -/* Reusable scratch for allocation-light repeated encoding. */ +/* Reusable scratch space for repeated encoding with fewer allocations. */ tok_wp_workspace_t *tok_wp_workspace_new(void); void tok_wp_workspace_free(tok_wp_workspace_t *ws); -/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1 - * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to - * the number of tokens needed/emitted in all cases. */ +/* Encode into caller-provided out_ids[0..out_cap-1]. + * Returns 0 on success, -1 on error, or -2 if out_cap is too small. + * If non-NULL, *out_n_tokens is always set to tokens needed/emitted. */ int tok_wp_encode_into(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text, @@ -58,8 +60,8 @@ int tok_wp_encode_into(const tok_wp_t *tok, int out_cap, int *out_n_tokens); -/* Same tokenization as tok_wp_encode(), reusing ws internally and - * performing only the final returned int[] allocation. */ +/* Same tokenization as tok_wp_encode(). + * Reuses ws internally and only allocates the returned int[]. */ int *tok_wp_encode_with_workspace(const tok_wp_t *tok, tok_wp_workspace_t *ws, const char *text, |
