1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
#include "encoder.h"
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char **argv) {
if (argc < 3) {
fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]);
return 1;
}
char error[256] = {0};
langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error));
if (!tokenizer) {
fprintf(stderr, "%s\n", error);
return 1;
}
for (int i = 2; i < argc; i++) {
int count = 0;
int *ids = langdet_tokenize(tokenizer, argv[i], &count);
if (!ids) {
fprintf(stderr, "Tokenization failed for argument %d\n", i);
langdet_tok_free(tokenizer);
return 1;
}
putchar('[');
for (int j = 0; j < count; j++)
printf("%s%d", j ? "," : "", ids[j]);
puts("]");
free(ids);
}
langdet_tok_free(tokenizer);
return fflush(stdout) == 0 ? 0 : 1;
}
|