aboutsummaryrefslogtreecommitdiff
path: root/tests/wordpiece_dump.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-27 00:58:05 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-27 01:03:41 +0200
commitaf55803d0dff9eb692c760c8d486e736c945061d (patch)
tree0875a1112bf98304e445e0544c4309670d6f4769 /tests/wordpiece_dump.c
parent0f861b7f3098750ddee4f6570f3be228d1da5320 (diff)
downloadffwd-af55803d0dff9eb692c760c8d486e736c945061d.tar.gz
ffwd-af55803d0dff9eb692c760c8d486e736c945061d.zip
chore: clean up comment formatting and wording
Diffstat (limited to 'tests/wordpiece_dump.c')
-rw-r--r--tests/wordpiece_dump.c15
1 files changed, 10 insertions, 5 deletions
diff --git a/tests/wordpiece_dump.c b/tests/wordpiece_dump.c
index 5dfb563..952e9e2 100644
--- a/tests/wordpiece_dump.c
+++ b/tests/wordpiece_dump.c
@@ -1,8 +1,13 @@
-/* Parity driver for the WordPiece tokenizer. Loads a BERT vocab dir, encodes
- * each argument through both encode entry points (and cross-checks they agree),
- * then prints "<count> <id> <id> ..." per line. check_wordpiece_parity.py builds
- * this with `make parity-wordpiece-driver` and compares the ids against the
- * stored reference vectors in tests/test-vectors/wordpiece/. Links libtok. */
+/* WordPiece tokenizer parity driver.
+ * Loads a BERT vocab dir, encodes each argument through both encode entry
+ * points, and checks that they agree.
+ *
+ * Prints one line per argument: "<count> <id> <id> ...".
+ *
+ * check_wordpiece_parity.py builds this with `make parity-wordpiece-driver` and
+ * compares ids against reference vectors in tests/test-vectors/wordpiece/.
+ */
+
#include "wordpiece.h"
#include <stdio.h>