aboutsummaryrefslogtreecommitdiff
path: root/libffwd/wordpiece.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-27 00:58:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-27 01:03:41 +0200
commitaf55803d0dff9eb692c760c8d486e736c945061d (patch)
tree0875a1112bf98304e445e0544c4309670d6f4769 /libffwd/wordpiece.h
parent0f861b7f3098750ddee4f6570f3be228d1da5320 (diff)
downloadffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz
ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip
chore: clean up comment formatting and wording
Diffstat (limited to 'libffwd/wordpiece.h')
-rw-r--r--libffwd/wordpiece.h44
1 files changed, 23 insertions, 21 deletions
diff --git a/libffwd/wordpiece.h b/libffwd/wordpiece.h
index c93140a..f8a63d6 100644
--- a/libffwd/wordpiece.h
+++ b/libffwd/wordpiece.h
@@ -1,15 +1,16 @@
/*
- * wordpiece.h - BERT-family WordPiece tokenizer.
+ * BERT-family WordPiece tokenizer.
*
- * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional
- * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting)
- * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes
- * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the
- * byte-level BPE backend leaves terminal/separator tokens to the frontend.
+ * Implements BERT tokenization: BasicTokenizer (cleanup, optional lowercase
+ * and accent stripping, CJK spacing, whitespace splitting, and punctuation
+ * splitting), then greedy longest-match WordPiece using vocab.txt.
*
- * The public surface mirrors bpe.h (load / encode with a reusable
- * workspace / decode / resolve a named special token / free) so both can sit
- * behind one tokenizer interface later.
+ * Encodes text only. The frontend adds [CLS]/[SEP], just as the byte-level BPE
+ * backend leaves terminal and separator tokens to the frontend.
+ *
+ * Public API mirrors bpe.h: load, encode with reusable workspace, decode,
+ * resolve named special tokens, and free. This lets both tokenizers share one
+ * tokenizer interface later.
*/
#ifndef TOK_WP_H
@@ -29,28 +30,29 @@ typedef struct {
typedef struct tok_wp_workspace tok_wp_workspace_t;
-/* Load from <model_dir>/vocab.txt, reading do_lower_case from
- * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on
- * error (missing vocab.txt, missing [UNK], allocation failure). */
+/* Load <model_dir>/vocab.txt and do_lower_case from
+ * <model_dir>/tokenizer_config.json; defaults to lowercase.
+ * Returns NULL on error: missing vocab.txt, missing [UNK], or allocation failure. */
tok_wp_t *tok_wp_load(const char *model_dir);
-/* Decode a single id to its vocab token string (internal pointer), or NULL. */
+/* Decode one id to its vocab token string, or NULL.
+ * Returns an internal pointer. */
const char *tok_wp_decode(const tok_wp_t *tok, int id);
/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
int tok_wp_token_id(const tok_wp_t *tok, const char *token);
-/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on
- * error (and *out_n_tokens = 0). */
+/* Encode UTF-8 text into a malloc'd id array and set *out_n_tokens.
+ * Returns NULL on error and sets *out_n_tokens = 0. */
int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens);
-/* Reusable scratch for allocation-light repeated encoding. */
+/* Reusable scratch space for repeated encoding with fewer allocations. */
tok_wp_workspace_t *tok_wp_workspace_new(void);
void tok_wp_workspace_free(tok_wp_workspace_t *ws);
-/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1
- * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
- * the number of tokens needed/emitted in all cases. */
+/* Encode into caller-provided out_ids[0..out_cap-1].
+ * Returns 0 on success, -1 on error, or -2 if out_cap is too small.
+ * If non-NULL, *out_n_tokens is always set to tokens needed/emitted. */
int tok_wp_encode_into(const tok_wp_t *tok,
tok_wp_workspace_t *ws,
const char *text,
@@ -58,8 +60,8 @@ int tok_wp_encode_into(const tok_wp_t *tok,
int out_cap,
int *out_n_tokens);
-/* Same tokenization as tok_wp_encode(), reusing ws internally and
- * performing only the final returned int[] allocation. */
+/* Same tokenization as tok_wp_encode().
+ * Reuses ws internally and only allocates the returned int[]. */
int *tok_wp_encode_with_workspace(const tok_wp_t *tok,
tok_wp_workspace_t *ws,
const char *text,