#include "encoder.h" #include #include int main(int argc, char **argv) { if (argc < 3) { fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]); return 1; } char error[256] = {0}; langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error)); if (!tokenizer) { fprintf(stderr, "%s\n", error); return 1; } for (int i = 2; i < argc; i++) { int count = 0; int *ids = langdet_tokenize(tokenizer, argv[i], &count); if (!ids) { fprintf(stderr, "Tokenization failed for argument %d\n", i); langdet_tok_free(tokenizer); return 1; } putchar('['); for (int j = 0; j < count; j++) printf("%s%d", j ? "," : "", ids[j]); puts("]"); free(ids); } langdet_tok_free(tokenizer); return fflush(stdout) == 0 ? 0 : 1; }