diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:04:04 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:04:04 +0200 |
| commit | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (patch) | |
| tree | 427e45b901aec902ceba72428fe5de9689d3001b /tools/token_ids.c | |
| parent | cb1a04331b27cc19445de63cedcd4fd30b8a9016 (diff) | |
| download | langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.tar.gz langdet-6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9.zip | |
Add CPU reference comparison and repeatable API tests
Diffstat (limited to 'tools/token_ids.c')
| -rw-r--r-- | tools/token_ids.c | 33 |
1 files changed, 33 insertions, 0 deletions
diff --git a/tools/token_ids.c b/tools/token_ids.c new file mode 100644 index 0000000..a8d9948 --- /dev/null +++ b/tools/token_ids.c @@ -0,0 +1,33 @@ +#include "encoder.h" + +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]); + return 1; + } + char error[256] = {0}; + langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error)); + if (!tokenizer) { + fprintf(stderr, "%s\n", error); + return 1; + } + for (int i = 2; i < argc; i++) { + int count = 0; + int *ids = langdet_tokenize(tokenizer, argv[i], &count); + if (!ids) { + fprintf(stderr, "Tokenization failed for argument %d\n", i); + langdet_tok_free(tokenizer); + return 1; + } + putchar('['); + for (int j = 0; j < count; j++) + printf("%s%d", j ? "," : "", ids[j]); + puts("]"); + free(ids); + } + langdet_tok_free(tokenizer); + return fflush(stdout) == 0 ? 0 : 1; +} |
