diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:40:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:40:12 +0200 |
| commit | cb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch) | |
| tree | 85edb98bacb5ce8da99245343f22fe58e594229b | |
| parent | 53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff) | |
| download | langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip | |
Expose complete language prediction API and simplify CLI
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | README.md | 41 | ||||
| -rw-r--r-- | examples/detect.c | 27 | ||||
| -rw-r--r-- | langdetect.c | 178 | ||||
| -rw-r--r-- | liblangdet/config.h | 2 | ||||
| -rw-r--r-- | liblangdet/encoder.h | 49 | ||||
| -rw-r--r-- | liblangdet/langdet.c | 197 | ||||
| -rw-r--r-- | liblangdet/langdet.h | 71 | ||||
| -rw-r--r-- | liblangdet/model_types.h | 2 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.c | 2 | ||||
| -rw-r--r-- | liblangdet/tokenize.c | 2 |
11 files changed, 366 insertions, 207 deletions
@@ -5,7 +5,7 @@ CFLAGS ?= -O3 -march=native CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread LDLIBS += -framework Accelerate -lm -pthread -CPU_SOURCES = model.c model_blas.c forward.c alloc.c config.c \ +CPU_SOURCES = langdet.c model.c model_blas.c forward.c alloc.c config.c \ tokenize.c sentencepiece.c safetensors.c \ act.c gemm.c norm.c attn.c threadpool.c generic.c LIB_SOURCES = $(addprefix liblangdet/,$(CPU_SOURCES)) deps/yyjson/yyjson.c @@ -1,7 +1,7 @@ # XLM-R language detection on macOS CPU A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the -liblangdet CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language +liblangdet prediction API, ARM NEON kernels, and Apple Accelerate BLAS. It returns language predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python runtime is required for inference. @@ -37,8 +37,8 @@ Use `--` before text that begins with a hyphen. Diagnostics go to stderr. ## Inference -`langdetect.c` loads and validates the four F32 classifier tensors and the -`id2label` mapping. The encoder accepts F32 XLM-R weights with exact GeLU activation; +`liblangdet/langdet.c` loads and validates the four F32 classifier tensors and the +`id2label` mapping. `langdetect.c` handles only command-line input and JSON output. The encoder accepts F32 XLM-R weights with exact GeLU activation; other model families and reduced-precision weights are not supported. The encoder uses first-token pooling without L2 normalization. The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer, @@ -56,20 +56,35 @@ of accuracy on ambiguous or unsupported text. ## CPU library -`make` builds both `langdetect` and the static archive `liblangdet.a`. The public -encoder and tokenizer API is declared in `liblangdet/langdet.h`, with the -`langdet_` prefix. Each encoder call returns the first token's final hidden state; -the language-classification head and JSON output remain in `langdetect.c`. +`make` builds both `langdetect` and the static archive `liblangdet.a`. Include +`liblangdet/langdet.h` to use the complete prediction API from C or C++: -Link a C application with: +- `langdet_open(model_dir, n_threads, err, errlen)` loads all inference resources. + Use `0` threads for automatic selection, or an explicit count from 1 to 16. +- `langdet_predict(detector, text, &result, err, errlen)` returns the best language, + its score, all 20 language scores, token count, and truncation flag. +- `langdet_free(detector)` releases the resources. Passing NULL is safe. + +Opening returns NULL on failure. Prediction returns 0 on success or -1 on failure +and clears the result on failure. The optional error buffer receives a message; +model-file readers may also print diagnostics to stderr. Result label strings +remain valid until the detector is freed. Numeric scores are stored in the result +and do not change when another prediction runs. + +Use one API call at a time per process, including when multiple detectors are +loaded. Internal CPU workers may cooperate on a prediction, but simultaneous calls +from application threads are not supported. + +`examples/detect.c` is a complete application using only the public header: ```sh -cc -Iliblangdet application.c liblangdet.a -framework Accelerate -lm -pthread -o application +cc -Iliblangdet examples/detect.c liblangdet.a -framework Accelerate -lm -pthread -o build/detect-example +./build/detect-example ./model 'This is an English sentence.' 'Bugün hava çok güzel.' ``` -The archive includes its JSON parser. No separate parser library is needed. -Models own their memory-mapped weights; free their workspaces before freeing the -model. A tokenizer or workspace must not be used concurrently by multiple calls. +The archive includes its JSON parser. No separate parser library or private +inference headers are needed. The encoder, tokenizer, classifier, and thread +configuration are managed by the detector. ## Download @@ -102,6 +117,8 @@ byte offset. single-thread inference, and 512-token truncation. - Invalid thread counts, a missing model directory, and NUL bytes in stdin produce errors. +- Standalone C and C++ applications link using only `langdet.h` and the static + archive. Prediction results, error returns, and detector lifecycle checks pass. - The downloaded checkpoint files match Hugging Face's sizes and hashes. - Numerical equivalence with Hugging Face Transformers has not been established. These smoke checks are not an accuracy benchmark or exhaustive tokenizer test. diff --git a/examples/detect.c b/examples/detect.c new file mode 100644 index 0000000..dedec64 --- /dev/null +++ b/examples/detect.c @@ -0,0 +1,27 @@ +#include "langdet.h" + +#include <stdio.h> + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "Usage: %s MODEL_DIRECTORY TEXT [TEXT ...]\n", argv[0]); + return 1; + } + char error[256]; + langdet_t *detector = langdet_open(argv[1], 0, error, sizeof(error)); + if (!detector) { + fprintf(stderr, "%s\n", error); + return 1; + } + for (int i = 2; i < argc; i++) { + langdet_result_t result; + if (langdet_predict(detector, argv[i], &result, error, sizeof(error)) != 0) { + fprintf(stderr, "%s\n", error); + langdet_free(detector); + return 1; + } + printf("%s\t%.6f\n", result.label, result.score); + } + langdet_free(detector); + return 0; +} diff --git a/langdetect.c b/langdetect.c index 6418d2d..76d74ab 100644 --- a/langdetect.c +++ b/langdetect.c @@ -1,177 +1,48 @@ -#include "model_internal.h" -#include "kernels.h" -#include "threadpool.h" +#include "langdet.h" #include <yyjson.h> #include <errno.h> #include <getopt.h> -#include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <time.h> -#define HIDDEN_SIZE 768 -#define NUM_LABELS 20 -#define MAX_TOKENS 512 - -typedef struct { - langdet_model_t *model; - langdet_workspace_t *workspace; - langdet_tok_t *tokenizer; - yyjson_doc *config; - const char *labels[NUM_LABELS]; - const float *dense_weight, *dense_bias; - const float *out_weight, *out_bias; -} detector_t; - static double now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1000.0 + t.tv_nsec / 1e6; } -static const float *load_tensor(detector_t *d, const char *name, const int64_t *shape, int ndim) { - safetensors_file_t *file = NULL; - const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file); - if (!tensor) { - fprintf(stderr, "Missing classifier tensor: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) - return NULL; - /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */ - return safetensors_data(file, tensor); -} - -static void detector_free(detector_t *d) { - langdet_tok_free(d->tokenizer); - langdet_workspace_free(d->workspace); - langdet_model_free(d->model); - yyjson_doc_free(d->config); -} - -static int detector_load(detector_t *d, const char *directory) { - char path[1024]; - int length = snprintf(path, sizeof(path), "%s/config.json", directory); - if (length < 0 || (size_t)length >= sizeof(path)) { - fprintf(stderr, "Model path is too long\n"); - return -1; - } - yyjson_read_err error; - d->config = yyjson_read_file(path, 0, NULL, &error); - if (!d->config) { - fprintf(stderr, "Cannot read %s: %s\n", path, error.msg); - return -1; - } - yyjson_val *root = yyjson_doc_get_root(d->config); - if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || - !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0), - "XLMRobertaForSequenceClassification")) { - fprintf(stderr, "Expected an XLMRobertaForSequenceClassification checkpoint\n"); - return -1; - } - yyjson_val *labels = yyjson_obj_get(root, "id2label"); - if (yyjson_obj_size(labels) != NUM_LABELS) { - fprintf(stderr, "Expected %d language labels in config.json\n", NUM_LABELS); - return -1; - } - for (int i = 0; i < NUM_LABELS; i++) { - char key[16]; - snprintf(key, sizeof(key), "%d", i); - d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key)); - if (!d->labels[i] || !d->labels[i][0]) { - fprintf(stderr, "Missing language label %d\n", i); - return -1; - } - } - - d->model = langdet_model_load(directory); - if (!d->model) - return -1; - const langdet_config_t *cfg = langdet_model_config(d->model); - if (cfg->hidden_size != HIDDEN_SIZE || - cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) { - fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n"); - return -1; - } - d->dense_weight = load_tensor(d, "classifier.dense.weight", - (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2); - d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1); - d->out_weight = load_tensor(d, "classifier.out_proj.weight", - (int64_t[]){NUM_LABELS, HIDDEN_SIZE}, 2); - d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){NUM_LABELS}, 1); - if (!d->dense_weight || !d->dense_bias || !d->out_weight || !d->out_bias) - return -1; - - char message[256] = {0}; - d->tokenizer = langdet_tok_open(directory, message, sizeof(message)); - if (!d->tokenizer) { - fprintf(stderr, "Tokenizer: %s\n", message); - return -1; - } - d->workspace = langdet_workspace_new(d->model); - if (!d->workspace) { - fprintf(stderr, "Cannot allocate inference workspace\n"); - return -1; - } - return 0; -} - -static int classify(detector_t *d, const char *text, int all_scores) { +static int print_prediction(langdet_t *detector, const char *text, int all_scores) { double start = now_ms(); - int count = 0; - int *ids = langdet_tokenize(d->tokenizer, text, &count); - if (!ids || count < 2) { - fprintf(stderr, "Tokenization failed\n"); - free(ids); + langdet_result_t result; + char error[256]; + if (langdet_predict(detector, text, &result, error, sizeof(error)) != 0) { + fprintf(stderr, "%s\n", error); return -1; } - int truncated = count > MAX_TOKENS; - if (truncated) { - ids[MAX_TOKENS - 1] = ids[count - 1]; - count = MAX_TOKENS; - } - float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS]; - int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); - free(ids); - if (result != 0) { - fprintf(stderr, "Encoder inference failed\n"); - return -1; - } - linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE); - for (int i = 0; i < HIDDEN_SIZE; i++) - dense[i] = tanhf(dense[i]); - linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, NUM_LABELS); - softmax(scores, 1, NUM_LABELS); - int best = 0; - for (int i = 0; i < NUM_LABELS; i++) { - if (!isfinite(scores[i])) { - fprintf(stderr, "Classifier produced a non-finite score\n"); - return -1; - } - if (scores[i] > scores[best]) - best = i; - } double elapsed = now_ms() - start; yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - if (!doc) + if (!doc) { + fprintf(stderr, "Cannot allocate JSON output\n"); return -1; + } yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); int ok = root && - yyjson_mut_obj_add_str(doc, root, "label", d->labels[best]) && - yyjson_mut_obj_add_real(doc, root, "score", scores[best]) && - yyjson_mut_obj_add_int(doc, root, "tokens", count) && - yyjson_mut_obj_add_bool(doc, root, "truncated", truncated) && + yyjson_mut_obj_add_str(doc, root, "label", result.label) && + yyjson_mut_obj_add_real(doc, root, "score", result.score) && + yyjson_mut_obj_add_int(doc, root, "tokens", result.tokens) && + yyjson_mut_obj_add_bool(doc, root, "truncated", result.truncated) && yyjson_mut_obj_add_real(doc, root, "ms", elapsed); if (all_scores && ok) { yyjson_mut_val *all = yyjson_mut_obj(doc); ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all); - for (int i = 0; i < NUM_LABELS && ok; i++) - ok = yyjson_mut_obj_add_real(doc, all, d->labels[i], scores[i]); + for (int i = 0; i < LANGDET_NUM_LABELS && ok; i++) + ok = yyjson_mut_obj_add_real(doc, all, result.labels[i], result.scores[i]); } char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL; int written = json ? puts(json) : EOF; @@ -199,9 +70,7 @@ static void usage(FILE *out, const char *program) { int main(int argc, char **argv) { const char *directory = "model"; - int threads = tp_get_num_cpus(); - if (threads > 16) - threads = 16; + int threads = 0; int all_scores = 0; const struct option options[] = { {"all", no_argument, NULL, 'a'}, @@ -228,16 +97,16 @@ int main(int argc, char **argv) { default: usage(stderr, argv[0]); return 1; } } - detector_t detector = {0}; - if (detector_load(&detector, directory) != 0) { - detector_free(&detector); + char error[256]; + langdet_t *detector = langdet_open(directory, threads, error, sizeof(error)); + if (!detector) { + fprintf(stderr, "%s\n", error); return 1; } - tp_set_threads(threads); int status = 0; if (optind < argc) { for (int i = optind; i < argc && status == 0; i++) - status = classify(&detector, argv[i], all_scores); + status = print_prediction(detector, argv[i], all_scores); } else { char *line = NULL; size_t capacity = 0; @@ -250,7 +119,7 @@ int main(int argc, char **argv) { } while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r')) line[--length] = '\0'; - status = classify(&detector, line, all_scores); + status = print_prediction(detector, line, all_scores); if (status != 0) break; } @@ -260,7 +129,6 @@ int main(int argc, char **argv) { } free(line); } - tp_set_threads(1); - detector_free(&detector); + langdet_free(detector); return status == 0 ? 0 : 1; } diff --git a/liblangdet/config.h b/liblangdet/config.h index 79522fc..9caaa24 100644 --- a/liblangdet/config.h +++ b/liblangdet/config.h @@ -1,7 +1,7 @@ #ifndef LANGDET_CONFIG_H #define LANGDET_CONFIG_H -#include "langdet.h" +#include "encoder.h" int langdet_config_parse(langdet_config_t *cfg, const char *model_dir); diff --git a/liblangdet/encoder.h b/liblangdet/encoder.h new file mode 100644 index 0000000..7d954df --- /dev/null +++ b/liblangdet/encoder.h @@ -0,0 +1,49 @@ +#ifndef LANGDET_ENCODER_H +#define LANGDET_ENCODER_H + +#include <stddef.h> + +#define LANGDET_MAX_LAYERS 64 + +typedef struct { + int hidden_size; + int n_layers; + int n_heads; + int head_dim; + int intermediate_size; + int vocab_size; + int max_position_embeddings; + int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ + int type_vocab_size; + float layer_norm_eps; +} langdet_config_t; + +typedef struct langdet_model langdet_model_t; +typedef struct langdet_workspace langdet_workspace_t; +typedef struct langdet_tok langdet_tok_t; + +/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ +langdet_model_t *langdet_model_load(const char *model_dir); +void langdet_model_free(langdet_model_t *model); +const langdet_config_t *langdet_model_config(const langdet_model_t *model); + +/* A workspace belongs to one model and may serve only one inference call at a time. */ +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); +void langdet_workspace_free(langdet_workspace_t *ws); + +/* Write the first token's final hidden state to out_embedding[hidden_size]. + * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); +void langdet_tok_free(langdet_tok_t *t); + +/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. + * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); + +#endif /* LANGDET_ENCODER_H */ diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c new file mode 100644 index 0000000..9410ba2 --- /dev/null +++ b/liblangdet/langdet.c @@ -0,0 +1,197 @@ +#include "langdet.h" +#include "model_internal.h" +#include "kernels.h" +#include "threadpool.h" + +#include <yyjson.h> + +#include <math.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define HIDDEN_SIZE 768 + +struct langdet { + langdet_model_t *model; + langdet_workspace_t *workspace; + langdet_tok_t *tokenizer; + yyjson_doc *config; + const char *labels[LANGDET_NUM_LABELS]; + const float *dense_weight, *dense_bias; + const float *out_weight, *out_bias; + int n_threads; +}; + +static int set_error(char *err, size_t errlen, const char *format, ...) { + if (err && errlen) { + va_list args; + va_start(args, format); + vsnprintf(err, errlen, format, args); + va_end(args); + } + return -1; +} + +static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape, + int ndim, char *err, size_t errlen) { + safetensors_file_t *file = NULL; + const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file); + if (!tensor) { + set_error(err, errlen, "Missing classifier tensor: %s", name); + return NULL; + } + if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) { + set_error(err, errlen, "Invalid classifier tensor: %s", name); + return NULL; + } + /* Classifier pointers borrow the model's mapping until detector teardown. */ + return safetensors_data(file, tensor); +} + +static int detector_load(langdet_t *d, const char *directory, char *err, size_t errlen) { + char path[1024]; + int length = snprintf(path, sizeof(path), "%s/config.json", directory); + if (length < 0 || (size_t)length >= sizeof(path)) + return set_error(err, errlen, "Model path is too long"); + + yyjson_read_err error; + d->config = yyjson_read_file(path, 0, NULL, &error); + if (!d->config) + return set_error(err, errlen, "Cannot read %s: %s", path, error.msg); + + yyjson_val *root = yyjson_doc_get_root(d->config); + if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || + !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0), + "XLMRobertaForSequenceClassification")) + return set_error(err, errlen, "Expected an XLMRobertaForSequenceClassification checkpoint"); + + yyjson_val *labels = yyjson_obj_get(root, "id2label"); + if (yyjson_obj_size(labels) != LANGDET_NUM_LABELS) + return set_error(err, errlen, "Expected %d language labels in config.json", LANGDET_NUM_LABELS); + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + char key[16]; + snprintf(key, sizeof(key), "%d", i); + d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key)); + if (!d->labels[i] || !d->labels[i][0]) + return set_error(err, errlen, "Missing language label %d", i); + } + + d->model = langdet_model_load(directory); + if (!d->model) + return set_error(err, errlen, "Cannot load XLM-R encoder from %s", directory); + const langdet_config_t *cfg = langdet_model_config(d->model); + if (cfg->hidden_size != HIDDEN_SIZE || + cfg->max_position_embeddings - cfg->position_id_offset < LANGDET_MAX_TOKENS) + return set_error(err, errlen, "Expected a 768-dimensional XLM-R encoder with 512-token capacity"); + + d->dense_weight = load_tensor(d, "classifier.dense.weight", + (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2, err, errlen); + if (!d->dense_weight) + return -1; + d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1, err, errlen); + if (!d->dense_bias) + return -1; + d->out_weight = load_tensor(d, "classifier.out_proj.weight", + (int64_t[]){LANGDET_NUM_LABELS, HIDDEN_SIZE}, 2, err, errlen); + if (!d->out_weight) + return -1; + d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){LANGDET_NUM_LABELS}, 1, + err, errlen); + if (!d->out_bias) + return -1; + + d->tokenizer = langdet_tok_open(directory, err, errlen); + if (!d->tokenizer) + return -1; + d->workspace = langdet_workspace_new(d->model); + if (!d->workspace) + return set_error(err, errlen, "Cannot allocate inference workspace"); + return 0; +} + +langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen) { + if (err && errlen) + err[0] = '\0'; + if (!model_dir || n_threads < 0 || n_threads > 16) { + set_error(err, errlen, "Expected a model directory and a thread count between 0 and 16"); + return NULL; + } + langdet_t *d = calloc(1, sizeof(*d)); + if (!d) { + set_error(err, errlen, "Cannot allocate detector"); + return NULL; + } + d->n_threads = n_threads ? n_threads : tp_get_num_cpus(); + if (d->n_threads > 16) + d->n_threads = 16; + if (detector_load(d, model_dir, err, errlen) != 0) { + langdet_free(d); + return NULL; + } + tp_set_threads(d->n_threads); + return d; +} + +int langdet_predict(langdet_t *d, const char *text, langdet_result_t *result, + char *err, size_t errlen) { + if (err && errlen) + err[0] = '\0'; + if (result) + memset(result, 0, sizeof(*result)); + if (!d || !text || !result) + return set_error(err, errlen, "Detector, text, and result must not be NULL"); + + if (tp_num_threads() != d->n_threads) + tp_set_threads(d->n_threads); + int count = 0; + int *ids = langdet_tokenize(d->tokenizer, text, &count); + if (!ids || count < 2) { + free(ids); + return set_error(err, errlen, "Tokenization failed"); + } + int truncated = count > LANGDET_MAX_TOKENS; + if (truncated) { + ids[LANGDET_MAX_TOKENS - 1] = ids[count - 1]; + count = LANGDET_MAX_TOKENS; + } + float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[LANGDET_NUM_LABELS]; + int rc = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); + free(ids); + if (rc != 0) + return set_error(err, errlen, "Encoder inference failed"); + + linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE); + for (int i = 0; i < HIDDEN_SIZE; i++) + dense[i] = tanhf(dense[i]); + linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, LANGDET_NUM_LABELS); + softmax(scores, 1, LANGDET_NUM_LABELS); + int best = 0; + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + if (!isfinite(scores[i])) + return set_error(err, errlen, "Classifier produced a non-finite score"); + if (scores[i] > scores[best]) + best = i; + } + result->label = d->labels[best]; + result->score = scores[best]; + result->tokens = count; + result->truncated = truncated; + for (int i = 0; i < LANGDET_NUM_LABELS; i++) { + result->labels[i] = d->labels[i]; + result->scores[i] = scores[i]; + } + return 0; +} + +void langdet_free(langdet_t *d) { + if (!d) + return; + tp_set_threads(1); + langdet_tok_free(d->tokenizer); + langdet_workspace_free(d->workspace); + langdet_model_free(d->model); + yyjson_doc_free(d->config); + free(d); +} diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h index 9c24264..22e0b64 100644 --- a/liblangdet/langdet.h +++ b/liblangdet/langdet.h @@ -3,47 +3,48 @@ #include <stddef.h> -#define LANGDET_MAX_LAYERS 64 +#ifdef __cplusplus +extern "C" { +#endif -typedef struct { - int hidden_size; - int n_layers; - int n_heads; - int head_dim; - int intermediate_size; - int vocab_size; - int max_position_embeddings; - int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ - int type_vocab_size; - float layer_norm_eps; -} langdet_config_t; +#define LANGDET_NUM_LABELS 20 +#define LANGDET_MAX_TOKENS 512 + +typedef struct langdet langdet_t; -typedef struct langdet_model langdet_model_t; -typedef struct langdet_workspace langdet_workspace_t; -typedef struct langdet_tok langdet_tok_t; +typedef struct { + const char *label; + float score; + int tokens; + int truncated; + /* Label pointers belong to the detector and remain valid until langdet_free(). + * Scores are stored in this result, in the checkpoint's id2label order. */ + const char *labels[LANGDET_NUM_LABELS]; + float scores[LANGDET_NUM_LABELS]; +} langdet_result_t; -/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ -langdet_model_t *langdet_model_load(const char *model_dir); -void langdet_model_free(langdet_model_t *model); -const langdet_config_t *langdet_model_config(const langdet_model_t *model); +/* Library calls must not overlap across application threads, even for different + * handles. Internal CPU workers cooperate on one prediction at a time. */ -/* A workspace belongs to one model and may serve only one inference call at a time. */ -langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); -void langdet_workspace_free(langdet_workspace_t *ws); +/* Load the model, tokenizer, classifier, and labels from model_dir. + * n_threads is 1..16, or 0 to detect the CPU count and cap it at 16. + * Return NULL on failure. An optional err[errlen] receives a NUL-terminated + * message on failure and is cleared on success. Model-file readers may also + * write diagnostics to stderr. */ +langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen); -/* Write the first token's final hidden state to out_embedding[hidden_size]. - * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ -int langdet_model_encode_into(const langdet_model_t *model, - langdet_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding); +/* Classify NUL-terminated UTF-8 text. Truncate to 512 tokens, retaining </s>. + * Empty text is accepted but does not yield a meaningful language prediction. + * Return 0 on success, or -1 with a zeroed result on failure. The optional + * error buffer follows the same contract as langdet_open(). */ +int langdet_predict(langdet_t *detector, const char *text, langdet_result_t *result, + char *err, size_t errlen); -langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); -void langdet_tok_free(langdet_tok_t *t); +/* Release the detector and its resources. NULL is accepted. */ +void langdet_free(langdet_t *detector); -/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. - * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ -int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); +#ifdef __cplusplus +} +#endif #endif /* LANGDET_H */ diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h index 2170909..31d101a 100644 --- a/liblangdet/model_types.h +++ b/liblangdet/model_types.h @@ -1,7 +1,7 @@ #ifndef LANGDET_MODEL_TYPES_H #define LANGDET_MODEL_TYPES_H -#include "langdet.h" +#include "encoder.h" #include "safetensors.h" typedef struct { diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c index 6c30ba1..33efab8 100644 --- a/liblangdet/sentencepiece.c +++ b/liblangdet/sentencepiece.c @@ -856,7 +856,7 @@ int *tok_spm_encode_with_workspace(const tok_spm_t *tok, const char *text, int *out_n_tokens) { if (out_n_tokens) - *out_n_tokens = 0; + *out_n_tokens = -1; if (!tok || !ws || !text) return NULL; int n = 0; diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c index 5e26a18..96a5f42 100644 --- a/liblangdet/tokenize.c +++ b/liblangdet/tokenize.c @@ -1,4 +1,4 @@ -#include "langdet.h" +#include "encoder.h" #include "sentencepiece.h" #include <limits.h> |
