diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:40:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:40:12 +0200 |
| commit | cb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch) | |
| tree | 85edb98bacb5ce8da99245343f22fe58e594229b /liblangdet/langdet.c | |
| parent | 53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff) | |
| download | langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip | |
Expose complete language prediction API and simplify CLI
Diffstat (limited to 'liblangdet/langdet.c')
| -rw-r--r-- | liblangdet/langdet.c | 197 |
1 files changed, 197 insertions, 0 deletions
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c new file mode 100644 index 0000000..9410ba2 --- /dev/null +++ b/liblangdet/langdet.c @@ -0,0 +1,197 @@ +#include "langdet.h" +#include "model_internal.h" +#include "kernels.h" +#include "threadpool.h" + +#include <yyjson.h> + +#include <math.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define HIDDEN_SIZE 768 + +struct langdet { + langdet_model_t *model; + langdet_workspace_t *workspace; + langdet_tok_t *tokenizer; + yyjson_doc *config; + const char *labels[LANGDET_NUM_LABELS]; + const float *dense_weight, *dense_bias; + const float *out_weight, *out_bias; + int n_threads; +}; + +static int set_error(char *err, size_t errlen, const char *format, ...) { + if (err && errlen) { + va_list args; + va_start(args, format); + vsnprintf(err, errlen, format, args); + va_end(args); + } + return -1; +} + +static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape, + int ndim, char *err, size_t errlen) { + safetensors_file_t *file = NULL; + const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file); + if (!tensor) { + set_error(err, errlen, "Missing classifier tensor: %s", name); + return NULL; + } + if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) { + set_error(err, errlen, "Invalid classifier tensor: %s", name); + return NULL; + } + /* Classifier pointers borrow the model's mapping until detector teardown. */ + return safetensors_data(file, tensor); +} + +static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) { + char path[1024]; + int length = snprintf(path, sizeof(path), "%s/config.json", directory); + if (length < 0 || (size_t)length >= sizeof(path)) + return set_error(err, errlen, "Model path is too long"); + + yyjson_read_err error; + d->config = yyjson_read_file(path, 0, NULL, &error); + if (!d->config) + return set_error(err, errlen, "Cannot read %s: %s", path, error.msg); + + yyjson_val *root = yyjson_doc_get_root(d->config); + if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || + !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0), + "XLMRobertaForSequenceClassification")) + return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint"); + + yyjson_val *labels = yyjson_obj_get(root, "id2label"); + if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS) + return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS); + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + char key[16]; + snprintf(key, sizeof(key), "%d", i); + d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key)); + if (!d->labels[i] || !d->labels[i][0]) + return set_error(err, errlen, "Missing language label %d", i); + } + + d->model = langdet_model_load(directory); + if (!d->model) + return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory); + const langdet_config_t *cfg = langdet_model_config(d->model); + if (cfg->hidden_size != HIDDEN_SIZE || + cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS) + return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity"); + + d->dense_weight = load_tensor(d, "classifier.dense.weight", + (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen); + if (!d->dense_weight) + return -1; + d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen); + if (!d->dense_bias) + return -1; + d->out_weight = load_tensor(d, "classifier.out_proj.weight", + (int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen); + if (!d->out_weight) + return -1; + d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1, + err, errlen); + if (!d->out_bias) + return -1; + + d->tokenizer = langdet_tok_open(directory, err, errlen); + if (!d->tokenizer) + return -1; + d->workspace = langdet_workspace_new(d->model); + if (!d->workspace) + return set_error(err, errlen, "Cannot allocate inference workspace"); + return 0; +} + +langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) { + if (err && errlen) + err[0] = '\0'; + if (!model_dir || n_threads < 0 || n_threads > 16) { + set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16"); + return NULL; + } + langdet_t *d = calloc(1, sizeof(*d)); + if (!d) { + set_error(err, errlen, "Cannot allocate detector"); + return NULL; + } + d->n_threads = n_threads ? n_threads : tp_get_num_cpus(); + if (d->n_threads > 16) + d->n_threads = 16; + if (detector_load(d, model_dir, err, errlen) != 0) { + langdet_free(d); + return NULL; + } + tp_set_threads(d->n_threads); + return d; +} + +int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result, + char *err, size_t errlen) { + if (err && errlen) + err[0] = '\0'; + if (result) + memset(result, 0, sizeof(*result)); + if (!d || !text || !result) + return set_error(err, errlen, "Detector, text, and result must not be NULL"); + + if (tp_num_threads() != d->n_threads) + tp_set_threads(d->n_threads); + int count = 0; + int *ids = langdet_tokenize(d->tokenizer, text, &count); + if (!ids || count < 2) { + free(ids); + return set_error(err, errlen, "Tokenization failed"); + } + int truncated = count > LANGDET_MAX_TOKENS; + if (truncated) { + ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1]; + count = LANGDET_MAX_TOKENS; + } + float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS]; + int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); + free(ids); + if (rc != 0) + return set_error(err, errlen, "Encoder inference failed"); + + linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE); + for (int i = 0; i < HIDDEN_SIZE; i++) + dense[i] = tanhf(dense[i]); + linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS); + softmax(scores, 1, LANGDET_NUM_LABELS); + int best = 0; + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + if (!isfinite(scores[i])) + return set_error(err, errlen, "Classifier produced a non-finite score"); + if (scores[i] > scores[best]) + best = i; + } + result->label = d->labels[best]; + result->score = scores[best]; + result->tokens = count; + result->truncated = truncated; + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + result->labels[i] = d->labels[i]; + result->scores[i] = scores[i]; + } + return 0; +} + +void langdet_free(langdet_t *d) { + if (!d) + return; + tp_set_threads(1); + langdet_tok_free(d->tokenizer); + langdet_workspace_free(d->workspace); + langdet_model_free(d->model); + yyjson_doc_free(d->config); + free(d); +} |
