1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
#include "langdet.h"
#include "model_internal.h"
#include "kernels.h"
#include "threadpool.h"
#include <yyjson.h>
#include <math.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define HIDDEN_SIZE 768
struct langdet {
langdet_model_t *model;
langdet_workspace_t *workspace;
langdet_tok_t *tokenizer;
yyjson_doc *config;
const char *labels[LANGDET_NUM_LABELS];
const float *dense_weight, *dense_bias;
const float *out_weight, *out_bias;
int n_threads;
};
static int set_error(char *err, size_t errlen, const char *format, ...) {
if (err && errlen) {
va_list args;
va_start(args, format);
vsnprintf(err, errlen, format, args);
va_end(args);
}
return -1;
}
static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape,
int ndim, char *err, size_t errlen) {
const safetensors_file_t *file = d->model->safetensors;
const safetensor_t *tensor = safetensors_find(file, name);
if (!tensor) {
set_error(err, errlen, "Missing classifier tensor: %s", name);
return NULL;
}
if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) {
set_error(err, errlen, "Invalid classifier tensor: %s", name);
return NULL;
}
/* Classifier pointers borrow the model's mapping until detector teardown. */
return safetensors_data(file, tensor);
}
static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) {
char path[1024];
int length = snprintf(path, sizeof(path), "%s/config.json", directory);
if (length < 0 || (size_t)length >= sizeof(path))
return set_error(err, errlen, "Model path is too long");
yyjson_read_err error;
d->config = yyjson_read_file(path, 0, NULL, &error);
if (!d->config)
return set_error(err, errlen, "Cannot read %s: %s", path, error.msg);
yyjson_val *root = yyjson_doc_get_root(d->config);
if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
!yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
"XLMRobertaForSequenceClassification"))
return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint");
yyjson_val *labels = yyjson_obj_get(root, "id2label");
if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS)
return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS);
for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
char key[16];
snprintf(key, sizeof(key), "%d", i);
d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
if (!d->labels[i] || !d->labels[i][0])
return set_error(err, errlen, "Missing language label %d", i);
}
d->model = langdet_model_load(directory);
if (!d->model)
return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory);
const langdet_config_t *cfg = langdet_model_config(d->model);
if (cfg->hidden_size != HIDDEN_SIZE ||
cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS)
return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity");
d->dense_weight = load_tensor(d, "classifier.dense.weight",
(int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen);
if (!d->dense_weight)
return -1;
d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen);
if (!d->dense_bias)
return -1;
d->out_weight = load_tensor(d, "classifier.out_proj.weight",
(int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen);
if (!d->out_weight)
return -1;
d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1,
err, errlen);
if (!d->out_bias)
return -1;
d->tokenizer = langdet_tok_open(directory, err, errlen);
if (!d->tokenizer)
return -1;
d->workspace = langdet_workspace_new(d->model);
if (!d->workspace)
return set_error(err, errlen, "Cannot allocate inference workspace");
return 0;
}
langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) {
if (err && errlen)
err[0] = '\0';
if (!model_dir || n_threads < 0 || n_threads > 16) {
set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16");
return NULL;
}
langdet_t *d = calloc(1, sizeof(*d));
if (!d) {
set_error(err, errlen, "Cannot allocate detector");
return NULL;
}
d->n_threads = n_threads ? n_threads : tp_get_num_cpus();
if (d->n_threads > 16)
d->n_threads = 16;
if (detector_load(d, model_dir, err, errlen) != 0) {
langdet_free(d);
return NULL;
}
tp_set_threads(d->n_threads);
return d;
}
int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result,
char *err, size_t errlen) {
if (err && errlen)
err[0] = '\0';
if (result)
memset(result, 0, sizeof(*result));
if (!d || !text || !result)
return set_error(err, errlen, "Detector, text, and result must not be NULL");
tp_set_threads(d->n_threads);
int count = 0;
int *ids = langdet_tokenize(d->tokenizer, text, &count);
if (!ids || count < 2) {
free(ids);
return set_error(err, errlen, "Tokenization failed");
}
int truncated = count > LANGDET_MAX_TOKENS;
if (truncated) {
ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1];
count = LANGDET_MAX_TOKENS;
}
float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS];
int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
free(ids);
if (rc != 0)
return set_error(err, errlen, "Encoder inference failed");
linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
for (int i = 0; i < HIDDEN_SIZE; i++)
dense[i] = tanhf(dense[i]);
linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS);
softmax(scores, 1, LANGDET_NUM_LABELS);
int best = 0;
for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
if (!isfinite(scores[i]))
return set_error(err, errlen, "Classifier produced a non-finite score");
if (scores[i] > scores[best])
best = i;
}
result->label = d->labels[best];
result->score = scores[best];
result->tokens = count;
result->truncated = truncated;
for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
result->labels[i] = d->labels[i];
result->scores[i] = scores[i];
}
return 0;
}
void langdet_free(langdet_t *d) {
if (!d)
return;
tp_set_threads(1);
langdet_tok_free(d->tokenizer);
langdet_workspace_free(d->workspace);
langdet_model_free(d->model);
yyjson_doc_free(d->config);
free(d);
}
|