aboutsummaryrefslogtreecommitdiff
path: root/tools/token_ids.c
blob: a8d99486d7affcc94450753bff6da53dce15d256 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
#include "encoder.h"

#include <stdio.h>
#include <stdlib.h>

int main(int argc, char **argv) {
    if (argc < 3) {
        fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]);
        return 1;
    }
    char error[256] = {0};
    langdet_tok_t *tokenizer = langdet_tok_open(argv[1], error, sizeof(error));
    if (!tokenizer) {
        fprintf(stderr, "%s\n", error);
        return 1;
    }
    for (int i = 2; i < argc; i++) {
        int count = 0;
        int *ids = langdet_tokenize(tokenizer, argv[i], &count);
        if (!ids) {
            fprintf(stderr, "Tokenization failed for argument %d\n", i);
            langdet_tok_free(tokenizer);
            return 1;
        }
        putchar('[');
        for (int j = 0; j < count; j++)
            printf("%s%d", j ? "," : "", ids[j]);
        puts("]");
        free(ids);
    }
    langdet_tok_free(tokenizer);
    return fflush(stdout) == 0 ? 0 : 1;
}