aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/langdet.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
commitcb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch)
tree85edb98bacb5ce8da99245343f22fe58e594229b /liblangdet/langdet.c
parent53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff)
downloadlangdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz
langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip
Expose complete language prediction API and simplify CLI
Diffstat (limited to 'liblangdet/langdet.c')
-rw-r--r--liblangdet/langdet.c197
1 files changed, 197 insertions, 0 deletions
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c
new file mode 100644
index 0000000..9410ba2
--- /dev/null
+++ b/liblangdet/langdet.c
@@ -0,0 +1,197 @@
+#include "langdet.h"
+#include "model_internal.h"
+#include "kernels.h"
+#include "threadpool.h"
+
+#include <yyjson.h>
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define HIDDEN_SIZE 768
+
+struct langdet {
+ langdet_model_t *model;
+ langdet_workspace_t *workspace;
+ langdet_tok_t *tokenizer;
+ yyjson_doc *config;
+ const char *labels[LANGDET_NUM_LABELS];
+ const float *dense_weight, *dense_bias;
+ const float *out_weight, *out_bias;
+ int n_threads;
+};
+
+static int set_error(char *err, size_t errlen, const char *format, ...) {
+ if (err && errlen) {
+ va_list args;
+ va_start(args, format);
+ vsnprintf(err, errlen, format, args);
+ va_end(args);
+ }
+ return -1;
+}
+
+static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape,
+ int ndim, char *err, size_t errlen) {
+ safetensors_file_t *file = NULL;
+ const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
+ if (!tensor) {
+ set_error(err, errlen, "Missing classifier tensor: %s", name);
+ return NULL;
+ }
+ if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) {
+ set_error(err, errlen, "Invalid classifier tensor: %s", name);
+ return NULL;
+ }
+ /* Classifier pointers borrow the model's mapping until detector teardown. */
+ return safetensors_data(file, tensor);
+}
+
+static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) {
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/config.json", directory);
+ if (length < 0 || (size_t)length >= sizeof(path))
+ return set_error(err, errlen, "Model path is too long");
+
+ yyjson_read_err error;
+ d->config = yyjson_read_file(path, 0, NULL, &error);
+ if (!d->config)
+ return set_error(err, errlen, "Cannot read %s: %s", path, error.msg);
+
+ yyjson_val *root = yyjson_doc_get_root(d->config);
+ if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
+ !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
+ "XLMRobertaForSequenceClassification"))
+ return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint");
+
+ yyjson_val *labels = yyjson_obj_get(root, "id2label");
+ if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS)
+ return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS);
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ char key[16];
+ snprintf(key, sizeof(key), "%d", i);
+ d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
+ if (!d->labels[i] || !d->labels[i][0])
+ return set_error(err, errlen, "Missing language label %d", i);
+ }
+
+ d->model = langdet_model_load(directory);
+ if (!d->model)
+ return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory);
+ const langdet_config_t *cfg = langdet_model_config(d->model);
+ if (cfg->hidden_size != HIDDEN_SIZE ||
+ cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS)
+ return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity");
+
+ d->dense_weight = load_tensor(d, "classifier.dense.weight",
+ (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen);
+ if (!d->dense_weight)
+ return -1;
+ d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen);
+ if (!d->dense_bias)
+ return -1;
+ d->out_weight = load_tensor(d, "classifier.out_proj.weight",
+ (int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen);
+ if (!d->out_weight)
+ return -1;
+ d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1,
+ err, errlen);
+ if (!d->out_bias)
+ return -1;
+
+ d->tokenizer = langdet_tok_open(directory, err, errlen);
+ if (!d->tokenizer)
+ return -1;
+ d->workspace = langdet_workspace_new(d->model);
+ if (!d->workspace)
+ return set_error(err, errlen, "Cannot allocate inference workspace");
+ return 0;
+}
+
+langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) {
+ if (err && errlen)
+ err[0] = '\0';
+ if (!model_dir || n_threads < 0 || n_threads > 16) {
+ set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16");
+ return NULL;
+ }
+ langdet_t *d = calloc(1, sizeof(*d));
+ if (!d) {
+ set_error(err, errlen, "Cannot allocate detector");
+ return NULL;
+ }
+ d->n_threads = n_threads ? n_threads : tp_get_num_cpus();
+ if (d->n_threads > 16)
+ d->n_threads = 16;
+ if (detector_load(d, model_dir, err, errlen) != 0) {
+ langdet_free(d);
+ return NULL;
+ }
+ tp_set_threads(d->n_threads);
+ return d;
+}
+
+int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result,
+ char *err, size_t errlen) {
+ if (err && errlen)
+ err[0] = '\0';
+ if (result)
+ memset(result, 0, sizeof(*result));
+ if (!d || !text || !result)
+ return set_error(err, errlen, "Detector, text, and result must not be NULL");
+
+ if (tp_num_threads() != d->n_threads)
+ tp_set_threads(d->n_threads);
+ int count = 0;
+ int *ids = langdet_tokenize(d->tokenizer, text, &count);
+ if (!ids || count < 2) {
+ free(ids);
+ return set_error(err, errlen, "Tokenization failed");
+ }
+ int truncated = count > LANGDET_MAX_TOKENS;
+ if (truncated) {
+ ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1];
+ count = LANGDET_MAX_TOKENS;
+ }
+ float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS];
+ int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
+ free(ids);
+ if (rc != 0)
+ return set_error(err, errlen, "Encoder inference failed");
+
+ linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
+ for (int i = 0; i < HIDDEN_SIZE; i++)
+ dense[i] = tanhf(dense[i]);
+ linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS);
+ softmax(scores, 1, LANGDET_NUM_LABELS);
+ int best = 0;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ if (!isfinite(scores[i]))
+ return set_error(err, errlen, "Classifier produced a non-finite score");
+ if (scores[i] > scores[best])
+ best = i;
+ }
+ result->label = d->labels[best];
+ result->score = scores[best];
+ result->tokens = count;
+ result->truncated = truncated;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ result->labels[i] = d->labels[i];
+ result->scores[i] = scores[i];
+ }
+ return 0;
+}
+
+void langdet_free(langdet_t *d) {
+ if (!d)
+ return;
+ tp_set_threads(1);
+ langdet_tok_free(d->tokenizer);
+ langdet_workspace_free(d->workspace);
+ langdet_model_free(d->model);
+ yyjson_doc_free(d->config);
+ free(d);
+}