aboutsummaryrefslogtreecommitdiff
path: root/tools/token_ids.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:04:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:04:04 +0200
commit6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (patch)
tree427e45b901aec902ceba72428fe5de9689d3001b /tools/token_ids.c
parentcb1a04331b27cc19445de63cedcd4fd30b8a9016 (diff)
downloadlangdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.tar.gz
langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.zip
Add CPU reference comparison and repeatable API tests
Diffstat (limited to 'tools/token_ids.c')
-rw-r--r--tools/token_ids.c33
1 files changed, 33 insertions, 0 deletions
diff --git a/tools/token_ids.c b/tools/token_ids.c
new file mode 100644
index 0000000..a8d9948
--- /dev/null
+++ b/tools/token_ids.c
@@ -0,0 +1,33 @@
+#include "encoder.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+ if (argc < 3) {
+ fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]);
+ return 1;
+ }
+ char error[256] = {0};
+ langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error));
+ if (!tokenizer) {
+ fprintf(stderr, "%s\n", error);
+ return 1;
+ }
+ for (int i = 2; i < argc; i++) {
+ int count = 0;
+ int *ids = langdet_tokenize(tokenizer, argv[i], &count);
+ if (!ids) {
+ fprintf(stderr, "Tokenization failed for argument %d\n", i);
+ langdet_tok_free(tokenizer);
+ return 1;
+ }
+ putchar('[');
+ for (int j = 0; j < count; j++)
+ printf("%s%d", j ? "," : "", ids[j]);
+ puts("]");
+ free(ids);
+ }
+ langdet_tok_free(tokenizer);
+ return fflush(stdout) == 0 ? 0 : 1;
+}