aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
commitcb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch)
tree85edb98bacb5ce8da99245343f22fe58e594229b
parent53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff)
downloadlangdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz
langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip
Expose complete language prediction API and simplify CLI
-rw-r--r--Makefile2
-rw-r--r--README.md41
-rw-r--r--examples/detect.c27
-rw-r--r--langdetect.c178
-rw-r--r--liblangdet/config.h2
-rw-r--r--liblangdet/encoder.h49
-rw-r--r--liblangdet/langdet.c197
-rw-r--r--liblangdet/langdet.h71
-rw-r--r--liblangdet/model_types.h2
-rw-r--r--liblangdet/sentencepiece.c2
-rw-r--r--liblangdet/tokenize.c2
11 files changed, 366 insertions, 207 deletions
diff --git a/Makefile b/Makefile
index 4ac4306..eafcf26 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ CFLAGS ?= -O3 -march=native
CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread
LDLIBS += -framework Accelerate -lm -pthread
-CPU_SOURCES = model.c model_blas.c forward.c alloc.c config.c \
+CPU_SOURCES = langdet.c model.c model_blas.c forward.c alloc.c config.c \
tokenize.c sentencepiece.c safetensors.c \
act.c gemm.c norm.c attn.c threadpool.c generic.c
LIB_SOURCES = $(addprefix liblangdet/,$(CPU_SOURCES)) deps/yyjson/yyjson.c
diff --git a/README.md b/README.md
index e74eab9..bdab9bc 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
# XLM-R language detection on macOS CPU
A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the
-liblangdet CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language
+liblangdet prediction API, ARM NEON kernels, and Apple Accelerate BLAS. It returns language
predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python
runtime is required for inference.
@@ -37,8 +37,8 @@ Use `--` before text that begins with a hyphen. Diagnostics go to stderr.
## Inference
-`langdetect.c` loads and validates the four F32 classifier tensors and the
-`id2label` mapping. The encoder accepts F32 XLM-R weights with exact GeLU activation;
+`liblangdet/langdet.c` loads and validates the four F32 classifier tensors and the
+`id2label` mapping. `langdetect.c` handles only command-line input and JSON output. The encoder accepts F32 XLM-R weights with exact GeLU activation;
other model families and reduced-precision weights are not supported.
The encoder uses first-token pooling without L2 normalization.
The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer,
@@ -56,20 +56,35 @@ of accuracy on ambiguous or unsupported text.
## CPU library
-`make` builds both `langdetect` and the static archive `liblangdet.a`. The public
-encoder and tokenizer API is declared in `liblangdet/langdet.h`, with the
-`langdet_` prefix. Each encoder call returns the first token's final hidden state;
-the language-classification head and JSON output remain in `langdetect.c`.
+`make` builds both `langdetect` and the static archive `liblangdet.a`. Include
+`liblangdet/langdet.h` to use the complete prediction API from C or C++:
-Link a C application with:
+- `langdet_open(model_dir, n_threads, err, errlen)` loads all inference resources.
+ Use `0` threads for automatic selection, or an explicit count from 1 to 16.
+- `langdet_predict(detector, text, &result, err, errlen)` returns the best language,
+ its score, all 20 language scores, token count, and truncation flag.
+- `langdet_free(detector)` releases the resources. Passing NULL is safe.
+
+Opening returns NULL on failure. Prediction returns 0 on success or -1 on failure
+and clears the result on failure. The optional error buffer receives a message;
+model-file readers may also print diagnostics to stderr. Result label strings
+remain valid until the detector is freed. Numeric scores are stored in the result
+and do not change when another prediction runs.
+
+Use one API call at a time per process, including when multiple detectors are
+loaded. Internal CPU workers may cooperate on a prediction, but simultaneous calls
+from application threads are not supported.
+
+`examples/detect.c` is a complete application using only the public header:
```sh
-cc -Iliblangdet application.c liblangdet.a -framework Accelerate -lm -pthread -o application
+cc -Iliblangdet examples/detect.c liblangdet.a -framework Accelerate -lm -pthread -o build/detect-example
+./build/detect-example ./model 'This is an English sentence.' 'Bugün hava çok güzel.'
```
-The archive includes its JSON parser. No separate parser library is needed.
-Models own their memory-mapped weights; free their workspaces before freeing the
-model. A tokenizer or workspace must not be used concurrently by multiple calls.
+The archive includes its JSON parser. No separate parser library or private
+inference headers are needed. The encoder, tokenizer, classifier, and thread
+configuration are managed by the detector.
## Download
@@ -102,6 +117,8 @@ byte offset.
single-thread inference, and 512-token truncation.
- Invalid thread counts, a missing model directory, and NUL bytes in stdin
produce errors.
+- Standalone C and C++ applications link using only `langdet.h` and the static
+ archive. Prediction results, error returns, and detector lifecycle checks pass.
- The downloaded checkpoint files match Hugging Face's sizes and hashes.
- Numerical equivalence with Hugging Face Transformers has not been established.
These smoke checks are not an accuracy benchmark or exhaustive tokenizer test.
diff --git a/examples/detect.c b/examples/detect.c
new file mode 100644
index 0000000..dedec64
--- /dev/null
+++ b/examples/detect.c
@@ -0,0 +1,27 @@
+#include "langdet.h"
+
+#include <stdio.h>
+
+int main(int argc, char **argv) {
+ if (argc < 3) {
+ fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]);
+ return 1;
+ }
+ char error[256];
+ langdet_t *detector = langdet_open(argv[1], 0, error, sizeof(error));
+ if (!detector) {
+ fprintf(stderr, "%s\n", error);
+ return 1;
+ }
+ for (int i = 2; i < argc; i++) {
+ langdet_result_t result;
+ if (langdet_predict(detector, argv[i], &result, error, sizeof(error)) != 0) {
+ fprintf(stderr, "%s\n", error);
+ langdet_free(detector);
+ return 1;
+ }
+ printf("%s\t%.6f\n", result.label, result.score);
+ }
+ langdet_free(detector);
+ return 0;
+}
diff --git a/langdetect.c b/langdetect.c
index 6418d2d..76d74ab 100644
--- a/langdetect.c
+++ b/langdetect.c
@@ -1,177 +1,48 @@
-#include "model_internal.h"
-#include "kernels.h"
-#include "threadpool.h"
+#include "langdet.h"
#include <yyjson.h>
#include <errno.h>
#include <getopt.h>
-#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
-#define HIDDEN_SIZE 768
-#define NUM_LABELS 20
-#define MAX_TOKENS 512
-
-typedef struct {
- langdet_model_t *model;
- langdet_workspace_t *workspace;
- langdet_tok_t *tokenizer;
- yyjson_doc *config;
- const char *labels[NUM_LABELS];
- const float *dense_weight, *dense_bias;
- const float *out_weight, *out_bias;
-} detector_t;
-
static double now_ms(void) {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec * 1000.0 + t.tv_nsec / 1e6;
}
-static const float *load_tensor(detector_t *d, const char *name, const int64_t *shape, int ndim) {
- safetensors_file_t *file = NULL;
- const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
- if (!tensor) {
- fprintf(stderr, "Missing classifier tensor: %s\n", name);
- return NULL;
- }
- if (!tensor_has_supported_shape(file, tensor, name, shape, ndim))
- return NULL;
- /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */
- return safetensors_data(file, tensor);
-}
-
-static void detector_free(detector_t *d) {
- langdet_tok_free(d->tokenizer);
- langdet_workspace_free(d->workspace);
- langdet_model_free(d->model);
- yyjson_doc_free(d->config);
-}
-
-static int detector_load(detector_t *d, const char *directory) {
- char path[1024];
- int length = snprintf(path, sizeof(path), "%s/config.json", directory);
- if (length < 0 || (size_t)length >= sizeof(path)) {
- fprintf(stderr, "Model path is too long\n");
- return -1;
- }
- yyjson_read_err error;
- d->config = yyjson_read_file(path, 0, NULL, &error);
- if (!d->config) {
- fprintf(stderr, "Cannot read %s: %s\n", path, error.msg);
- return -1;
- }
- yyjson_val *root = yyjson_doc_get_root(d->config);
- if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
- !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
- "XLMRobertaForSequenceClassification")) {
- fprintf(stderr, "Expected an XLMRobertaForSequenceClassification checkpoint\n");
- return -1;
- }
- yyjson_val *labels = yyjson_obj_get(root, "id2label");
- if (yyjson_obj_size(labels) != NUM_LABELS) {
- fprintf(stderr, "Expected %d language labels in config.json\n", NUM_LABELS);
- return -1;
- }
- for (int i = 0; i < NUM_LABELS; i++) {
- char key[16];
- snprintf(key, sizeof(key), "%d", i);
- d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
- if (!d->labels[i] || !d->labels[i][0]) {
- fprintf(stderr, "Missing language label %d\n", i);
- return -1;
- }
- }
-
- d->model = langdet_model_load(directory);
- if (!d->model)
- return -1;
- const langdet_config_t *cfg = langdet_model_config(d->model);
- if (cfg->hidden_size != HIDDEN_SIZE ||
- cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) {
- fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n");
- return -1;
- }
- d->dense_weight = load_tensor(d, "classifier.dense.weight",
- (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2);
- d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1);
- d->out_weight = load_tensor(d, "classifier.out_proj.weight",
- (int64_t[]){NUM_LABELS, HIDDEN_SIZE}, 2);
- d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){NUM_LABELS}, 1);
- if (!d->dense_weight || !d->dense_bias || !d->out_weight || !d->out_bias)
- return -1;
-
- char message[256] = {0};
- d->tokenizer = langdet_tok_open(directory, message, sizeof(message));
- if (!d->tokenizer) {
- fprintf(stderr, "Tokenizer: %s\n", message);
- return -1;
- }
- d->workspace = langdet_workspace_new(d->model);
- if (!d->workspace) {
- fprintf(stderr, "Cannot allocate inference workspace\n");
- return -1;
- }
- return 0;
-}
-
-static int classify(detector_t *d, const char *text, int all_scores) {
+static int print_prediction(langdet_t *detector, const char *text, int all_scores) {
double start = now_ms();
- int count = 0;
- int *ids = langdet_tokenize(d->tokenizer, text, &count);
- if (!ids || count < 2) {
- fprintf(stderr, "Tokenization failed\n");
- free(ids);
+ langdet_result_t result;
+ char error[256];
+ if (langdet_predict(detector, text, &result, error, sizeof(error)) != 0) {
+ fprintf(stderr, "%s\n", error);
return -1;
}
- int truncated = count > MAX_TOKENS;
- if (truncated) {
- ids[MAX_TOKENS - 1] = ids[count - 1];
- count = MAX_TOKENS;
- }
- float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS];
- int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
- free(ids);
- if (result != 0) {
- fprintf(stderr, "Encoder inference failed\n");
- return -1;
- }
- linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
- for (int i = 0; i < HIDDEN_SIZE; i++)
- dense[i] = tanhf(dense[i]);
- linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, NUM_LABELS);
- softmax(scores, 1, NUM_LABELS);
- int best = 0;
- for (int i = 0; i < NUM_LABELS; i++) {
- if (!isfinite(scores[i])) {
- fprintf(stderr, "Classifier produced a non-finite score\n");
- return -1;
- }
- if (scores[i] > scores[best])
- best = i;
- }
double elapsed = now_ms() - start;
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
- if (!doc)
+ if (!doc) {
+ fprintf(stderr, "Cannot allocate JSON output\n");
return -1;
+ }
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
int ok = root &&
- yyjson_mut_obj_add_str(doc, root, "label", d->labels[best]) &&
- yyjson_mut_obj_add_real(doc, root, "score", scores[best]) &&
- yyjson_mut_obj_add_int(doc, root, "tokens", count) &&
- yyjson_mut_obj_add_bool(doc, root, "truncated", truncated) &&
+ yyjson_mut_obj_add_str(doc, root, "label", result.label) &&
+ yyjson_mut_obj_add_real(doc, root, "score", result.score) &&
+ yyjson_mut_obj_add_int(doc, root, "tokens", result.tokens) &&
+ yyjson_mut_obj_add_bool(doc, root, "truncated", result.truncated) &&
yyjson_mut_obj_add_real(doc, root, "ms", elapsed);
if (all_scores && ok) {
yyjson_mut_val *all = yyjson_mut_obj(doc);
ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all);
- for (int i = 0; i < NUM_LABELS && ok; i++)
- ok = yyjson_mut_obj_add_real(doc, all, d->labels[i], scores[i]);
+ for (int i = 0; i < LANGDET_NUM_LABELS && ok; i++)
+ ok = yyjson_mut_obj_add_real(doc, all, result.labels[i], result.scores[i]);
}
char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL;
int written = json ? puts(json) : EOF;
@@ -199,9 +70,7 @@ static void usage(FILE *out, const char *program) {
int main(int argc, char **argv) {
const char *directory = "model";
- int threads = tp_get_num_cpus();
- if (threads > 16)
- threads = 16;
+ int threads = 0;
int all_scores = 0;
const struct option options[] = {
{"all", no_argument, NULL, 'a'},
@@ -228,16 +97,16 @@ int main(int argc, char **argv) {
default: usage(stderr, argv[0]); return 1;
}
}
- detector_t detector = {0};
- if (detector_load(&detector, directory) != 0) {
- detector_free(&detector);
+ char error[256];
+ langdet_t *detector = langdet_open(directory, threads, error, sizeof(error));
+ if (!detector) {
+ fprintf(stderr, "%s\n", error);
return 1;
}
- tp_set_threads(threads);
int status = 0;
if (optind < argc) {
for (int i = optind; i < argc && status == 0; i++)
- status = classify(&detector, argv[i], all_scores);
+ status = print_prediction(detector, argv[i], all_scores);
} else {
char *line = NULL;
size_t capacity = 0;
@@ -250,7 +119,7 @@ int main(int argc, char **argv) {
}
while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r'))
line[--length] = '\0';
- status = classify(&detector, line, all_scores);
+ status = print_prediction(detector, line, all_scores);
if (status != 0)
break;
}
@@ -260,7 +129,6 @@ int main(int argc, char **argv) {
}
free(line);
}
- tp_set_threads(1);
- detector_free(&detector);
+ langdet_free(detector);
return status == 0 ? 0 : 1;
}
diff --git a/liblangdet/config.h b/liblangdet/config.h
index 79522fc..9caaa24 100644
--- a/liblangdet/config.h
+++ b/liblangdet/config.h
@@ -1,7 +1,7 @@
#ifndef LANGDET_CONFIG_H
#define LANGDET_CONFIG_H
-#include "langdet.h"
+#include "encoder.h"
int langdet_config_parse(langdet_config_t *cfg, const char *model_dir);
diff --git a/liblangdet/encoder.h b/liblangdet/encoder.h
new file mode 100644
index 0000000..7d954df
--- /dev/null
+++ b/liblangdet/encoder.h
@@ -0,0 +1,49 @@
+#ifndef LANGDET_ENCODER_H
+#define LANGDET_ENCODER_H
+
+#include <stddef.h>
+
+#define LANGDET_MAX_LAYERS 64
+
+typedef struct {
+ int hidden_size;
+ int n_layers;
+ int n_heads;
+ int head_dim;
+ int intermediate_size;
+ int vocab_size;
+ int max_position_embeddings;
+ int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
+ int type_vocab_size;
+ float layer_norm_eps;
+} langdet_config_t;
+
+typedef struct langdet_model langdet_model_t;
+typedef struct langdet_workspace langdet_workspace_t;
+typedef struct langdet_tok langdet_tok_t;
+
+/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
+langdet_model_t *langdet_model_load(const char *model_dir);
+void langdet_model_free(langdet_model_t *model);
+const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+
+/* A workspace belongs to one model and may serve only one inference call at a time. */
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
+void langdet_workspace_free(langdet_workspace_t *ws);
+
+/* Write the first token's final hidden state to out_embedding[hidden_size].
+ * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
+
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
+void langdet_tok_free(langdet_tok_t *t);
+
+/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
+ * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+
+#endif /* LANGDET_ENCODER_H */
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c
new file mode 100644
index 0000000..9410ba2
--- /dev/null
+++ b/liblangdet/langdet.c
@@ -0,0 +1,197 @@
+#include "langdet.h"
+#include "model_internal.h"
+#include "kernels.h"
+#include "threadpool.h"
+
+#include <yyjson.h>
+
+#include <math.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define HIDDEN_SIZE 768
+
+struct langdet {
+ langdet_model_t *model;
+ langdet_workspace_t *workspace;
+ langdet_tok_t *tokenizer;
+ yyjson_doc *config;
+ const char *labels[LANGDET_NUM_LABELS];
+ const float *dense_weight, *dense_bias;
+ const float *out_weight, *out_bias;
+ int n_threads;
+};
+
+static int set_error(char *err, size_t errlen, const char *format, ...) {
+ if (err && errlen) {
+ va_list args;
+ va_start(args, format);
+ vsnprintf(err, errlen, format, args);
+ va_end(args);
+ }
+ return -1;
+}
+
+static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape,
+ int ndim, char *err, size_t errlen) {
+ safetensors_file_t *file = NULL;
+ const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
+ if (!tensor) {
+ set_error(err, errlen, "Missing classifier tensor: %s", name);
+ return NULL;
+ }
+ if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) {
+ set_error(err, errlen, "Invalid classifier tensor: %s", name);
+ return NULL;
+ }
+ /* Classifier pointers borrow the model's mapping until detector teardown. */
+ return safetensors_data(file, tensor);
+}
+
+static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) {
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/config.json", directory);
+ if (length < 0 || (size_t)length >= sizeof(path))
+ return set_error(err, errlen, "Model path is too long");
+
+ yyjson_read_err error;
+ d->config = yyjson_read_file(path, 0, NULL, &error);
+ if (!d->config)
+ return set_error(err, errlen, "Cannot read %s: %s", path, error.msg);
+
+ yyjson_val *root = yyjson_doc_get_root(d->config);
+ if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
+ !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
+ "XLMRobertaForSequenceClassification"))
+ return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint");
+
+ yyjson_val *labels = yyjson_obj_get(root, "id2label");
+ if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS)
+ return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS);
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ char key[16];
+ snprintf(key, sizeof(key), "%d", i);
+ d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
+ if (!d->labels[i] || !d->labels[i][0])
+ return set_error(err, errlen, "Missing language label %d", i);
+ }
+
+ d->model = langdet_model_load(directory);
+ if (!d->model)
+ return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory);
+ const langdet_config_t *cfg = langdet_model_config(d->model);
+ if (cfg->hidden_size != HIDDEN_SIZE ||
+ cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS)
+ return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity");
+
+ d->dense_weight = load_tensor(d, "classifier.dense.weight",
+ (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen);
+ if (!d->dense_weight)
+ return -1;
+ d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen);
+ if (!d->dense_bias)
+ return -1;
+ d->out_weight = load_tensor(d, "classifier.out_proj.weight",
+ (int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen);
+ if (!d->out_weight)
+ return -1;
+ d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1,
+ err, errlen);
+ if (!d->out_bias)
+ return -1;
+
+ d->tokenizer = langdet_tok_open(directory, err, errlen);
+ if (!d->tokenizer)
+ return -1;
+ d->workspace = langdet_workspace_new(d->model);
+ if (!d->workspace)
+ return set_error(err, errlen, "Cannot allocate inference workspace");
+ return 0;
+}
+
+langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) {
+ if (err && errlen)
+ err[0] = '\0';
+ if (!model_dir || n_threads < 0 || n_threads > 16) {
+ set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16");
+ return NULL;
+ }
+ langdet_t *d = calloc(1, sizeof(*d));
+ if (!d) {
+ set_error(err, errlen, "Cannot allocate detector");
+ return NULL;
+ }
+ d->n_threads = n_threads ? n_threads : tp_get_num_cpus();
+ if (d->n_threads > 16)
+ d->n_threads = 16;
+ if (detector_load(d, model_dir, err, errlen) != 0) {
+ langdet_free(d);
+ return NULL;
+ }
+ tp_set_threads(d->n_threads);
+ return d;
+}
+
+int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result,
+ char *err, size_t errlen) {
+ if (err && errlen)
+ err[0] = '\0';
+ if (result)
+ memset(result, 0, sizeof(*result));
+ if (!d || !text || !result)
+ return set_error(err, errlen, "Detector, text, and result must not be NULL");
+
+ if (tp_num_threads() != d->n_threads)
+ tp_set_threads(d->n_threads);
+ int count = 0;
+ int *ids = langdet_tokenize(d->tokenizer, text, &count);
+ if (!ids || count < 2) {
+ free(ids);
+ return set_error(err, errlen, "Tokenization failed");
+ }
+ int truncated = count > LANGDET_MAX_TOKENS;
+ if (truncated) {
+ ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1];
+ count = LANGDET_MAX_TOKENS;
+ }
+ float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS];
+ int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
+ free(ids);
+ if (rc != 0)
+ return set_error(err, errlen, "Encoder inference failed");
+
+ linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
+ for (int i = 0; i < HIDDEN_SIZE; i++)
+ dense[i] = tanhf(dense[i]);
+ linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS);
+ softmax(scores, 1, LANGDET_NUM_LABELS);
+ int best = 0;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ if (!isfinite(scores[i]))
+ return set_error(err, errlen, "Classifier produced a non-finite score");
+ if (scores[i] > scores[best])
+ best = i;
+ }
+ result->label = d->labels[best];
+ result->score = scores[best];
+ result->tokens = count;
+ result->truncated = truncated;
+ for (int i = 0; i < LANGDET_NUM_LABELS; i++) {
+ result->labels[i] = d->labels[i];
+ result->scores[i] = scores[i];
+ }
+ return 0;
+}
+
+void langdet_free(langdet_t *d) {
+ if (!d)
+ return;
+ tp_set_threads(1);
+ langdet_tok_free(d->tokenizer);
+ langdet_workspace_free(d->workspace);
+ langdet_model_free(d->model);
+ yyjson_doc_free(d->config);
+ free(d);
+}
diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h
index 9c24264..22e0b64 100644
--- a/liblangdet/langdet.h
+++ b/liblangdet/langdet.h
@@ -3,47 +3,48 @@
#include <stddef.h>
-#define LANGDET_MAX_LAYERS 64
+#ifdef __cplusplus
+extern "C" {
+#endif
-typedef struct {
- int hidden_size;
- int n_layers;
- int n_heads;
- int head_dim;
- int intermediate_size;
- int vocab_size;
- int max_position_embeddings;
- int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
- int type_vocab_size;
- float layer_norm_eps;
-} langdet_config_t;
+#define LANGDET_NUM_LABELS 20
+#define LANGDET_MAX_TOKENS 512
+
+typedef struct langdet langdet_t;
-typedef struct langdet_model langdet_model_t;
-typedef struct langdet_workspace langdet_workspace_t;
-typedef struct langdet_tok langdet_tok_t;
+typedef struct {
+ const char *label;
+ float score;
+ int tokens;
+ int truncated;
+ /* Label pointers belong to the detector and remain valid until langdet_free().
+ * Scores are stored in this result, in the checkpoint's id2label order. */
+ const char *labels[LANGDET_NUM_LABELS];
+ float scores[LANGDET_NUM_LABELS];
+} langdet_result_t;
-/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
-langdet_model_t *langdet_model_load(const char *model_dir);
-void langdet_model_free(langdet_model_t *model);
-const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+/* Library calls must not overlap across application threads, even for different
+ * handles. Internal CPU workers cooperate on one prediction at a time. */
-/* A workspace belongs to one model and may serve only one inference call at a time. */
-langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
-void langdet_workspace_free(langdet_workspace_t *ws);
+/* Load the model, tokenizer, classifier, and labels from model_dir.
+ * n_threads is 1..16, or 0 to detect the CPU count and cap it at 16.
+ * Return NULL on failure. An optional err[errlen] receives a NUL-terminated
+ * message on failure and is cleared on success. Model-file readers may also
+ * write diagnostics to stderr. */
+langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen);
-/* Write the first token's final hidden state to out_embedding[hidden_size].
- * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
-int langdet_model_encode_into(const langdet_model_t *model,
- langdet_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
+/* Classify NUL-terminated UTF-8 text. Truncate to 512 tokens, retaining </s>.
+ * Empty text is accepted but does not yield a meaningful language prediction.
+ * Return 0 on success, or -1 with a zeroed result on failure. The optional
+ * error buffer follows the same contract as langdet_open(). */
+int langdet_predict(langdet_t *detector, const char *text, langdet_result_t *result,
+ char *err, size_t errlen);
-langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
-void langdet_tok_free(langdet_tok_t *t);
+/* Release the detector and its resources. NULL is accepted. */
+void langdet_free(langdet_t *detector);
-/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
- * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
-int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+#ifdef __cplusplus
+}
+#endif
#endif /* LANGDET_H */
diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h
index 2170909..31d101a 100644
--- a/liblangdet/model_types.h
+++ b/liblangdet/model_types.h
@@ -1,7 +1,7 @@
#ifndef LANGDET_MODEL_TYPES_H
#define LANGDET_MODEL_TYPES_H
-#include "langdet.h"
+#include "encoder.h"
#include "safetensors.h"
typedef struct {
diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c
index 6c30ba1..33efab8 100644
--- a/liblangdet/sentencepiece.c
+++ b/liblangdet/sentencepiece.c
@@ -856,7 +856,7 @@ int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
const char *text,
int *out_n_tokens) {
if (out_n_tokens)
- *out_n_tokens = 0;
+ *out_n_tokens = -1;
if (!tok || !ws || !text)
return NULL;
int n = 0;
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
index 5e26a18..96a5f42 100644
--- a/liblangdet/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -1,4 +1,4 @@
-#include "langdet.h"
+#include "encoder.h"
#include "sentencepiece.h"
#include <limits.h>