aboutsummaryrefslogtreecommitdiff
path: root/tests/wordpiece_dump.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/wordpiece_dump.c
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/wordpiece_dump.c')
-rw-r--r--tests/wordpiece_dump.c52
1 files changed, 52 insertions, 0 deletions
diff --git a/tests/wordpiece_dump.c b/tests/wordpiece_dump.c
new file mode 100644
index 0000000..5dfb563
--- /dev/null
+++ b/tests/wordpiece_dump.c
@@ -0,0 +1,52 @@
+/* Parity driver for the WordPiece tokenizer. Loads a BERT vocab dir, encodes
+ * each argument through both encode entry points (and cross-checks they agree),
+ * then prints "<count> <id> <id> ..." per line. check_wordpiece_parity.py builds
+ * this with `make parity-wordpiece-driver` and compares the ids against the
+ * stored reference vectors in tests/test-vectors/wordpiece/. Links libtok. */
+#include "wordpiece.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char **argv) {
+ if (argc < 2) {
+ fprintf(stderr, "usage: %s MODEL_DIR [TEXT...]\n", argv[0]);
+ return 2;
+ }
+ tok_wp_t *tok = tok_wp_load(argv[1]);
+ if (!tok) {
+ fprintf(stderr, "failed to load tokenizer: %s\n", argv[1]);
+ return 1;
+ }
+ tok_wp_workspace_t *ws = tok_wp_workspace_new();
+ for (int a = 2; a < argc; a++) {
+ int n = 0;
+ int *ids = tok_wp_encode(tok, argv[a], &n);
+ if (!ids && n != 0) {
+ fprintf(stderr, "encode failed for argument %d\n", a - 2);
+ return 1;
+ }
+ int n_ws = 0;
+ int *ids_ws = tok_wp_encode_with_workspace(tok, ws, argv[a], &n_ws);
+ if (n_ws != n) {
+ fprintf(stderr, "workspace count mismatch for argument %d\n", a - 2);
+ return 1;
+ }
+ for (int i = 0; i < n; i++) {
+ if (ids[i] != ids_ws[i]) {
+ fprintf(stderr, "workspace id mismatch for argument %d\n", a - 2);
+ return 1;
+ }
+ }
+ printf("%d", n);
+ for (int i = 0; i < n; i++)
+ printf(" %d", ids[i]);
+ printf("\n");
+ free(ids);
+ free(ids_ws);
+ }
+ tok_wp_workspace_free(ws);
+ tok_wp_free(tok);
+ return 0;
+}