From 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Tue, 29 Sep 2026 14:04:04 +0200 Subject: Add CPU reference comparison and repeatable API tests --- tools/token_ids.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tools/token_ids.c (limited to 'tools/token_ids.c') diff --git a/tools/token_ids.c b/tools/token_ids.c new file mode 100644 index 0000000..a8d9948 --- /dev/null +++ b/tools/token_ids.c @@ -0,0 +1,33 @@ +#include "encoder.h" + +#include +#include + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]); + return 1; + } + char error[256] = {0}; + langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error)); + if (!tokenizer) { + fprintf(stderr, "%s\n", error); + return 1; + } + for (int i = 2; i < argc; i++) { + int count = 0; + int *ids = langdet_tokenize(tokenizer, argv[i], &count); + if (!ids) { + fprintf(stderr, "Tokenization failed for argument %d\n", i); + langdet_tok_free(tokenizer); + return 1; + } + putchar('['); + for (int j = 0; j < count; j++) + printf("%s%d", j ? "," : "", ids[j]); + puts("]"); + free(ids); + } + langdet_tok_free(tokenizer); + return fflush(stdout) == 0 ? 0 : 1; +} -- cgit v1.3.1