aboutsummaryrefslogtreecommitdiff
path: root/liblangdet
diff options
context:
space:
mode:
Diffstat (limited to 'liblangdet')
-rw-r--r--liblangdet/config.h2
-rw-r--r--liblangdet/encoder.h49
-rw-r--r--liblangdet/langdet.c197
-rw-r--r--liblangdet/langdet.h71
-rw-r--r--liblangdet/model_types.h2
-rw-r--r--liblangdet/sentencepiece.c2
-rw-r--r--liblangdet/tokenize.c2
7 files changed, 286 insertions, 39 deletions
diff --git a/liblangdet/config.h b/liblangdet/config.h
index 79522fc..9caaa24 100644
--- a/liblangdet/config.h
+++ b/liblangdet/config.h
@@ -1,7 +1,7 @@
#ifndef LANGDET_CONFIG_H
#define LANGDET_CONFIG_H
-#include "langdet.h"
+#include "encoder.h"
int langdet_config_parse(langdet_config_t *cfg, const char *model_dir);
diff --git a/liblangdet/encoder.h b/liblangdet/encoder.h
new file mode 100644
index 0000000..7d954df
--- /dev/null
+++ b/liblangdet/encoder.h
@@ -0,0 +1,49 @@
+#ifndef LANGDET_ENCODER_H
+#define LANGDET_ENCODER_H
+
+#include <stddef.h>
+
+#define LANGDET_MAX_LAYERS 64
+
+typedef struct {
+ int hidden_size;
+ int n_layers;
+ int n_heads;
+ int head_dim;
+ int intermediate_size;
+ int vocab_size;
+ int max_position_embeddings;
+ int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
+ int type_vocab_size;
+ float layer_norm_eps;
+} langdet_config_t;
+
+typedef struct langdet_model langdet_model_t;
+typedef struct langdet_workspace langdet_workspace_t;
+typedef struct langdet_tok langdet_tok_t;
+
+/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
+langdet_model_t *langdet_model_load(const char *model_dir);
+void langdet_model_free(langdet_model_t *model);
+const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+
+/* A workspace belongs to one model and may serve only one inference call at a time. */
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
+void langdet_workspace_free(langdet_workspace_t *ws);
+
+/* Write the first token's final hidden state to out_embedding[hidden_size].
+ * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
+
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
+void langdet_tok_free(langdet_tok_t *t);
+
+/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
+ * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+
+#endif /* LANGDET_ENCODER_H */
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c
new file mode 100644
index 0000000..9410ba2
--- /dev/null
+++ b/liblangdet/langdet.c
@@ -0,0 +1,197 @@
+#include "langdet.h"
+#include "model_internal.h"
+#include "kernels.h"
+#include "threadpool.h"
+
+#include <yyjson.h>
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define HIDDEN_SIZE 768
+
+struct langdet {
+ langdet_model_t *model;
+ langdet_workspace_t *workspace;
+ langdet_tok_t *tokenizer;
+ yyjson_doc *config;
+ const char *labels[LANGDET_NUM_LABELS];
+ const float *dense_weight, *dense_bias;
+ const float *out_weight, *out_bias;
+ int n_threads;
+};
+
+static int set_error(char *err, size_t errlen, const char *format, ...) {
+ if (err && errlen) {
+ va_list args;
+ va_start(args, format);
+ vsnprintf(err, errlen, format, args);
+ va_end(args);
+ }
+ return -1;
+}
+
+static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape,
+ int ndim, char *err, size_t errlen) {
+ safetensors_file_t *file = NULL;
+ const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
+ if (!tensor) {
+ set_error(err, errlen, "Missing classifier tensor: %s", name);
+ return NULL;
+ }
+ if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) {
+ set_error(err, errlen, "Invalid classifier tensor: %s", name);
+ return NULL;
+ }
+ /* Classifier pointers borrow the model's mapping until detector teardown. */
+ return safetensors_data(file, tensor);
+}
+
+static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) {
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/config.json", directory);
+ if (length < 0 || (size_t)length >= sizeof(path))
+ return set_error(err, errlen, "Model path is too long");
+
+ yyjson_read_err error;
+ d->config = yyjson_read_file(path, 0, NULL, &error);
+ if (!d->config)
+ return set_error(err, errlen, "Cannot read %s: %s", path, error.msg);
+
+ yyjson_val *root = yyjson_doc_get_root(d->config);
+ if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
+ !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
+ "XLMRobertaForSequenceClassification"))
+ return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint");
+
+ yyjson_val *labels = yyjson_obj_get(root, "id2label");
+ if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS)
+ return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS);
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ char key[16];
+ snprintf(key, sizeof(key), "%d", i);
+ d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
+ if (!d->labels[i] || !d->labels[i][0])
+ return set_error(err, errlen, "Missing language label %d", i);
+ }
+
+ d->model = langdet_model_load(directory);
+ if (!d->model)
+ return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory);
+ const langdet_config_t *cfg = langdet_model_config(d->model);
+ if (cfg->hidden_size != HIDDEN_SIZE ||
+ cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS)
+ return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity");
+
+ d->dense_weight = load_tensor(d, "classifier.dense.weight",
+ (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen);
+ if (!d->dense_weight)
+ return -1;
+ d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen);
+ if (!d->dense_bias)
+ return -1;
+ d->out_weight = load_tensor(d, "classifier.out_proj.weight",
+ (int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen);
+ if (!d->out_weight)
+ return -1;
+ d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1,
+ err, errlen);
+ if (!d->out_bias)
+ return -1;
+
+ d->tokenizer = langdet_tok_open(directory, err, errlen);
+ if (!d->tokenizer)
+ return -1;
+ d->workspace = langdet_workspace_new(d->model);
+ if (!d->workspace)
+ return set_error(err, errlen, "Cannot allocate inference workspace");
+ return 0;
+}
+
+langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) {
+ if (err && errlen)
+ err[0] = '\0';
+ if (!model_dir || n_threads < 0 || n_threads > 16) {
+ set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16");
+ return NULL;
+ }
+ langdet_t *d = calloc(1, sizeof(*d));
+ if (!d) {
+ set_error(err, errlen, "Cannot allocate detector");
+ return NULL;
+ }
+ d->n_threads = n_threads ? n_threads : tp_get_num_cpus();
+ if (d->n_threads > 16)
+ d->n_threads = 16;
+ if (detector_load(d, model_dir, err, errlen) != 0) {
+ langdet_free(d);
+ return NULL;
+ }
+ tp_set_threads(d->n_threads);
+ return d;
+}
+
+int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result,
+ char *err, size_t errlen) {
+ if (err && errlen)
+ err[0] = '\0';
+ if (result)
+ memset(result, 0, sizeof(*result));
+ if (!d || !text || !result)
+ return set_error(err, errlen, "Detector, text, and result must not be NULL");
+
+ if (tp_num_threads() != d->n_threads)
+ tp_set_threads(d->n_threads);
+ int count = 0;
+ int *ids = langdet_tokenize(d->tokenizer, text, &count);
+ if (!ids || count < 2) {
+ free(ids);
+ return set_error(err, errlen, "Tokenization failed");
+ }
+ int truncated = count > LANGDET_MAX_TOKENS;
+ if (truncated) {
+ ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1];
+ count = LANGDET_MAX_TOKENS;
+ }
+ float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS];
+ int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
+ free(ids);
+ if (rc != 0)
+ return set_error(err, errlen, "Encoder inference failed");
+
+ linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
+ for (int i = 0; i < HIDDEN_SIZE; i++)
+ dense[i] = tanhf(dense[i]);
+ linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS);
+ softmax(scores, 1, LANGDET_NUM_LABELS);
+ int best = 0;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ if (!isfinite(scores[i]))
+ return set_error(err, errlen, "Classifier produced a non-finite score");
+ if (scores[i] > scores[best])
+ best = i;
+ }
+ result->label = d->labels[best];
+ result->score = scores[best];
+ result->tokens = count;
+ result->truncated = truncated;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ result->labels[i] = d->labels[i];
+ result->scores[i] = scores[i];
+ }
+ return 0;
+}
+
+void langdet_free(langdet_t *d) {
+ if (!d)
+ return;
+ tp_set_threads(1);
+ langdet_tok_free(d->tokenizer);
+ langdet_workspace_free(d->workspace);
+ langdet_model_free(d->model);
+ yyjson_doc_free(d->config);
+ free(d);
+}
diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h
index 9c24264..22e0b64 100644
--- a/liblangdet/langdet.h
+++ b/liblangdet/langdet.h
@@ -3,47 +3,48 @@
#include <stddef.h>
-#define LANGDET_MAX_LAYERS 64
+#ifdef __cplusplus
+extern "C" {
+#endif
-typedef struct {
- int hidden_size;
- int n_layers;
- int n_heads;
- int head_dim;
- int intermediate_size;
- int vocab_size;
- int max_position_embeddings;
- int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
- int type_vocab_size;
- float layer_norm_eps;
-} langdet_config_t;
+#define LANGDET_NUM_LABELS 20
+#define LANGDET_MAX_TOKENS 512
+
+typedef struct langdet langdet_t;
-typedef struct langdet_model langdet_model_t;
-typedef struct langdet_workspace langdet_workspace_t;
-typedef struct langdet_tok langdet_tok_t;
+typedef struct {
+ const char *label;
+ float score;
+ int tokens;
+ int truncated;
+ /* Label pointers belong to the detector and remain valid until langdet_free().
+ * Scores are stored in this result, in the checkpoint's id2label order. */
+ const char *labels[LANGDET_NUM_LABELS];
+ float scores[LANGDET_NUM_LABELS];
+} langdet_result_t;
-/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
-langdet_model_t *langdet_model_load(const char *model_dir);
-void langdet_model_free(langdet_model_t *model);
-const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+/* Library calls must not overlap across application threads, even for different
+ * handles. Internal CPU workers cooperate on one prediction at a time. */
-/* A workspace belongs to one model and may serve only one inference call at a time. */
-langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
-void langdet_workspace_free(langdet_workspace_t *ws);
+/* Load the model, tokenizer, classifier, and labels from model_dir.
+ * n_threads is 1..16, or 0 to detect the CPU count and cap it at 16.
+ * Return NULL on failure. An optional err[errlen] receives a NUL-terminated
+ * message on failure and is cleared on success. Model-file readers may also
+ * write diagnostics to stderr. */
+langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen);
-/* Write the first token's final hidden state to out_embedding[hidden_size].
- * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
-int langdet_model_encode_into(const langdet_model_t *model,
- langdet_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
+/* Classify NUL-terminated UTF-8 text. Truncate to 512 tokens, retaining </s>.
+ * Empty text is accepted but does not yield a meaningful language prediction.
+ * Return 0 on success, or -1 with a zeroed result on failure. The optional
+ * error buffer follows the same contract as langdet_open(). */
+int langdet_predict(langdet_t *detector, const char *text, langdet_result_t *result,
+ char *err, size_t errlen);
-langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
-void langdet_tok_free(langdet_tok_t *t);
+/* Release the detector and its resources. NULL is accepted. */
+void langdet_free(langdet_t *detector);
-/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
- * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
-int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+#ifdef __cplusplus
+}
+#endif
#endif /* LANGDET_H */
diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h
index 2170909..31d101a 100644
--- a/liblangdet/model_types.h
+++ b/liblangdet/model_types.h
@@ -1,7 +1,7 @@
#ifndef LANGDET_MODEL_TYPES_H
#define LANGDET_MODEL_TYPES_H
-#include "langdet.h"
+#include "encoder.h"
#include "safetensors.h"
typedef struct {
diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c
index 6c30ba1..33efab8 100644
--- a/liblangdet/sentencepiece.c
+++ b/liblangdet/sentencepiece.c
@@ -856,7 +856,7 @@ int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
const char *text,
int *out_n_tokens) {
if (out_n_tokens)
- *out_n_tokens = 0;
+ *out_n_tokens = -1;
if (!tok || !ws || !text)
return NULL;
int n = 0;
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
index 5e26a18..96a5f42 100644
--- a/liblangdet/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -1,4 +1,4 @@
-#include "langdet.h"
+#include "encoder.h"
#include "sentencepiece.h"
#include <limits.h>