1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#include "langdet.h"
#include <yyjson.h>
#include <errno.h>
#include <getopt.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
static double now_ms(void) {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec * 1000.0 + t.tv_nsec / 1e6;
}
static int print_prediction(langdet_t *detector, const char *text, int all_scores) {
double start = now_ms();
langdet_result_t result;
char error[256];
if (langdet_predict(detector, text, &result, error, sizeof(error)) != 0) {
fprintf(stderr, "%s\n", error);
return -1;
}
double elapsed = now_ms() - start;
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
if (!doc) {
fprintf(stderr, "Cannot allocate JSON output\n");
return -1;
}
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
int ok = root &&
yyjson_mut_obj_add_str(doc, root, "label", result.label) &&
yyjson_mut_obj_add_real(doc, root, "score", result.score) &&
yyjson_mut_obj_add_int(doc, root, "tokens", result.tokens) &&
yyjson_mut_obj_add_bool(doc, root, "truncated", result.truncated) &&
yyjson_mut_obj_add_real(doc, root, "ms", elapsed);
if (all_scores && ok) {
yyjson_mut_val *all = yyjson_mut_obj(doc);
ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all);
for (int i = 0; i < LANGDET_NUM_LABELS && ok; i++)
ok = yyjson_mut_obj_add_real(doc, all, result.labels[i], result.scores[i]);
}
char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL;
int written = json ? puts(json) : EOF;
free(json);
yyjson_mut_doc_free(doc);
if (written == EOF || fflush(stdout) == EOF) {
fprintf(stderr, "Cannot write classification result\n");
return -1;
}
return 0;
}
static void usage(FILE *out, const char *program) {
fprintf(out,
"Usage: %s [-d MODEL_DIR] [-t THREADS] [--all] [--] [TEXT ...]\n"
"Without TEXT arguments, read one UTF-8 text per line from stdin.\n"
"Write one JSON result per text, in input order.\n"
" -d DIR Model directory (default: ./model)\n"
" -t N CPU threads, 1-16 (default: detected CPU count, capped at 16)\n"
" --all Include scores for all 20 languages\n"
" -h Show help\n"
"Inputs longer than 512 tokens are truncated, retaining the final </s>.\n",
program);
}
int main(int argc, char **argv) {
const char *directory = "model";
int threads = 0;
int all_scores = 0;
const struct option options[] = {
{"all", no_argument, NULL, 'a'},
{"help", no_argument, NULL, 'h'},
{NULL, 0, NULL, 0},
};
int option;
while ((option = getopt_long(argc, argv, "d:t:ah", options, NULL)) != -1) {
switch (option) {
case 'd': directory = optarg; break;
case 'a': all_scores = 1; break;
case 'h': usage(stdout, argv[0]); return 0;
case 't': {
char *end;
errno = 0;
long n = strtol(optarg, &end, 10);
if (errno || *end || n < 1 || n > 16) {
fprintf(stderr, "Thread count must be an integer between 1 and 16\n");
return 1;
}
threads = (int)n;
break;
}
default: usage(stderr, argv[0]); return 1;
}
}
char error[256];
langdet_t *detector = langdet_open(directory, threads, error, sizeof(error));
if (!detector) {
fprintf(stderr, "%s\n", error);
return 1;
}
int status = 0;
if (optind < argc) {
for (int i = optind; i < argc && status == 0; i++)
status = print_prediction(detector, argv[i], all_scores);
} else {
char *line = NULL;
size_t capacity = 0;
ssize_t length;
while ((length = getline(&line, &capacity, stdin)) >= 0) {
if (memchr(line, '\0', (size_t)length)) {
fprintf(stderr, "Input must not contain NUL bytes\n");
status = -1;
break;
}
while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r'))
line[--length] = '\0';
status = print_prediction(detector, line, all_scores);
if (status != 0)
break;
}
if (ferror(stdin) || (!feof(stdin) && status == 0)) {
fprintf(stderr, "Cannot read input\n");
status = -1;
}
free(line);
}
langdet_free(detector);
return status == 0 ? 0 : 1;
}
|