aboutsummaryrefslogtreecommitdiff
path: root/libffwd/tokenize.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-27 00:58:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-27 01:03:41 +0200
commitaf55803d0dff9eb692c760c8d486e736c945061d (patch)
tree0875a1112bf98304e445e0544c4309670d6f4769 /libffwd/tokenize.c
parent0f861b7f3098750ddee4f6570f3be228d1da5320 (diff)
downloadffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz
ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip
chore: clean up comment formatting and wording
Diffstat (limited to 'libffwd/tokenize.c')
-rw-r--r--libffwd/tokenize.c14
1 files changed, 9 insertions, 5 deletions
diff --git a/libffwd/tokenize.c b/libffwd/tokenize.c
index df5861e..2af151a 100644
--- a/libffwd/tokenize.c
+++ b/libffwd/tokenize.c
@@ -1,10 +1,14 @@
/*
- * tokenize.c - model-aware tokenization: text -> the token ids a model expects.
+ * Model-aware tokenization: text -> token ids the model expects.
*
- * Backend-independent, so this is its own handle (ffwd_tok_t), not part of the
- * per-backend ffwd_t. ffwd_tok_open picks the tokenizer by the files present
- * and resolves the model's special-token ids; ffwd_tokenize applies that
- * model's layout (BERT/XLM-R [CLS]..[SEP], Qwen appended terminal token).
+ * Backend-independent, so this uses its own handle, ffwd_tok_t, instead of the
+ * per-backend ffwd_t.
+ *
+ * ffwd_tok_open selects the tokenizer from the files present and resolves the
+ * model's special-token ids.
+ *
+ * ffwd_tokenize applies the model's token layout: BERT/XLM-R [CLS]..[SEP], or
+ * Qwen appended terminal token.
*/
#include "ffwd.h"