aboutsummaryrefslogtreecommitdiff
path: root/tools/token_ids.c
diff options
context:
space:
mode:
Diffstat (limited to 'tools/token_ids.c')
-rw-r--r--tools/token_ids.c33
1 files changed, 33 insertions, 0 deletions
diff --git a/tools/token_ids.c b/tools/token_ids.c
new file mode 100644
index 0000000..a8d9948
--- /dev/null
+++ b/tools/token_ids.c
@@ -0,0 +1,33 @@
+#include "encoder.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv) {
+ if (argc < 3) {
+ fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]);
+ return 1;
+ }
+ char error[256] = {0};
+ langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error));
+ if (!tokenizer) {
+ fprintf(stderr, "%s\n", error);
+ return 1;
+ }
+ for (int i = 2; i < argc; i++) {
+ int count = 0;
+ int *ids = langdet_tokenize(tokenizer, argv[i], &count);
+ if (!ids) {
+ fprintf(stderr, "Tokenization failed for argument %d\n", i);
+ langdet_tok_free(tokenizer);
+ return 1;
+ }
+ putchar('[');
+ for (int j = 0; j < count; j++)
+ printf("%s%d", j ? "," : "", ids[j]);
+ puts("]");
+ free(ids);
+ }
+ langdet_tok_free(tokenizer);
+ return fflush(stdout) == 0 ? 0 : 1;
+}