diff options
Diffstat (limited to 'tools/token_ids.c')
| -rw-r--r-- | tools/token_ids.c | 33 |
1 files changed, 33 insertions, 0 deletions
diff --git a/tools/token_ids.c b/tools/token_ids.c new file mode 100644 index 0000000..a8d9948 --- /dev/null +++ b/tools/token_ids.c @@ -0,0 +1,33 @@ +#include "encoder.h" + +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]); + return 1; + } + char error[256] = {0}; + langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error)); + if (!tokenizer) { + fprintf(stderr, "%s\n", error); + return 1; + } + for (int i = 2; i < argc; i++) { + int count = 0; + int *ids = langdet_tokenize(tokenizer, argv[i], &count); + if (!ids) { + fprintf(stderr, "Tokenization failed for argument %d\n", i); + langdet_tok_free(tokenizer); + return 1; + } + putchar('['); + for (int j = 0; j < count; j++) + printf("%s%d", j ? "," : "", ids[j]); + puts("]"); + free(ids); + } + langdet_tok_free(tokenizer); + return fflush(stdout) == 0 ? 0 : 1; +} |
