#include "langdet.h" #include "model_internal.h" #include "kernels.h" #include "threadpool.h" #include #include #include #include #include #include #define HIDDEN_SIZE 768 struct langdet { langdet_model_t *model; langdet_workspace_t *workspace; langdet_tok_t *tokenizer; yyjson_doc *config; const char *labels[LANGDET_NUM_LABELS]; const float *dense_weight, *dense_bias; const float *out_weight, *out_bias; int n_threads; }; static int set_error(char *err, size_t errlen, const char *format, ...) { if (err && errlen) { va_list args; va_start(args, format); vsnprintf(err, errlen, format, args); va_end(args); } return -1; } static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape, int ndim, char *err, size_t errlen) { const safetensors_file_t *file = d->model->safetensors; const safetensor_t *tensor = safetensors_find(file, name); if (!tensor) { set_error(err, errlen, "Missing classifier tensor: %s", name); return NULL; } if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) { set_error(err, errlen, "Invalid classifier tensor: %s", name); return NULL; } /* Classifier pointers borrow the model's mapping until detector teardown. */ return safetensors_data(file, tensor); } static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) { char path[1024]; int length = snprintf(path, sizeof(path), "%s/config.json", directory); if (length < 0 || (size_t)length >= sizeof(path)) return set_error(err, errlen, "Model path is too long"); yyjson_read_err error; d->config = yyjson_read_file(path, 0, NULL, &error); if (!d->config) return set_error(err, errlen, "Cannot read %s: %s", path, error.msg); yyjson_val *root = yyjson_doc_get_root(d->config); if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0), "XLMRobertaForSequenceClassification")) return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint"); yyjson_val *labels = yyjson_obj_get(root, "id2label"); if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS) return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS); for (int i = 0; i < LANGDET_NUM_LABELS; i++) { char key[16]; snprintf(key, sizeof(key), "%d", i); d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key)); if (!d->labels[i] || !d->labels[i][0]) return set_error(err, errlen, "Missing language label %d", i); } d->model = langdet_model_load(directory); if (!d->model) return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory); const langdet_config_t *cfg = langdet_model_config(d->model); if (cfg->hidden_size != HIDDEN_SIZE || cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS) return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity"); d->dense_weight = load_tensor(d, "classifier.dense.weight", (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen); if (!d->dense_weight) return -1; d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen); if (!d->dense_bias) return -1; d->out_weight = load_tensor(d, "classifier.out_proj.weight", (int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen); if (!d->out_weight) return -1; d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1, err, errlen); if (!d->out_bias) return -1; d->tokenizer = langdet_tok_open(directory, err, errlen); if (!d->tokenizer) return -1; d->workspace = langdet_workspace_new(d->model); if (!d->workspace) return set_error(err, errlen, "Cannot allocate inference workspace"); return 0; } langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) { if (err && errlen) err[0] = '\0'; if (!model_dir || n_threads < 0 || n_threads > 16) { set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16"); return NULL; } langdet_t *d = calloc(1, sizeof(*d)); if (!d) { set_error(err, errlen, "Cannot allocate detector"); return NULL; } d->n_threads = n_threads ? n_threads : tp_get_num_cpus(); if (d->n_threads > 16) d->n_threads = 16; if (detector_load(d, model_dir, err, errlen) != 0) { langdet_free(d); return NULL; } tp_set_threads(d->n_threads); return d; } int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result, char *err, size_t errlen) { if (err && errlen) err[0] = '\0'; if (result) memset(result, 0, sizeof(*result)); if (!d || !text || !result) return set_error(err, errlen, "Detector, text, and result must not be NULL"); tp_set_threads(d->n_threads); int count = 0; int *ids = langdet_tokenize(d->tokenizer, text, &count); if (!ids || count < 2) { free(ids); return set_error(err, errlen, "Tokenization failed"); } int truncated = count > LANGDET_MAX_TOKENS; if (truncated) { ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1]; count = LANGDET_MAX_TOKENS; } float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS]; int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); free(ids); if (rc != 0) return set_error(err, errlen, "Encoder inference failed"); linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE); for (int i = 0; i < HIDDEN_SIZE; i++) dense[i] = tanhf(dense[i]); linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS); softmax(scores, 1, LANGDET_NUM_LABELS); int best = 0; for (int i = 0; i < LANGDET_NUM_LABELS; i++) { if (!isfinite(scores[i])) return set_error(err, errlen, "Classifier produced a non-finite score"); if (scores[i] > scores[best]) best = i; } result->label = d->labels[best]; result->score = scores[best]; result->tokens = count; result->truncated = truncated; for (int i = 0; i < LANGDET_NUM_LABELS; i++) { result->labels[i] = d->labels[i]; result->scores[i] = scores[i]; } return 0; } void langdet_free(langdet_t *d) { if (!d) return; tp_set_threads(1); langdet_tok_free(d->tokenizer); langdet_workspace_free(d->workspace); langdet_model_free(d->model); yyjson_doc_free(d->config); free(d); }