aboutsummaryrefslogtreecommitdiff
path: root/langdetect.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
commitcb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch)
tree85edb98bacb5ce8da99245343f22fe58e594229b /langdetect.c
parent53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff)
downloadlangdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz
langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip
Expose complete language prediction API and simplify CLI
Diffstat (limited to 'langdetect.c')
-rw-r--r--langdetect.c178
1 files changed, 23 insertions, 155 deletions
diff --git a/langdetect.c b/langdetect.c
index 6418d2d..76d74ab 100644
--- a/langdetect.c
+++ b/langdetect.c
@@ -1,177 +1,48 @@
-#include "model_internal.h"
-#include "kernels.h"
-#include "threadpool.h"
+#include "langdet.h"
#include <yyjson.h>
#include <errno.h>
#include <getopt.h>
-#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
-#define HIDDEN_SIZE 768
-#define NUM_LABELS 20
-#define MAX_TOKENS 512
-
-typedef struct {
- langdet_model_t *model;
- langdet_workspace_t *workspace;
- langdet_tok_t *tokenizer;
- yyjson_doc *config;
- const char *labels[NUM_LABELS];
- const float *dense_weight, *dense_bias;
- const float *out_weight, *out_bias;
-} detector_t;
-
static double now_ms(void) {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec * 1000.0 + t.tv_nsec / 1e6;
}
-static const float *load_tensor(detector_t *d, const char *name, const int64_t *shape, int ndim) {
- safetensors_file_t *file = NULL;
- const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
- if (!tensor) {
- fprintf(stderr, "Missing classifier tensor: %s\n", name);
- return NULL;
- }
- if (!tensor_has_supported_shape(file, tensor, name, shape, ndim))
- return NULL;
- /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */
- return safetensors_data(file, tensor);
-}
-
-static void detector_free(detector_t *d) {
- langdet_tok_free(d->tokenizer);
- langdet_workspace_free(d->workspace);
- langdet_model_free(d->model);
- yyjson_doc_free(d->config);
-}
-
-static int detector_load(detector_t *d, const char *directory) {
- char path[1024];
- int length = snprintf(path, sizeof(path), "%s/config.json", directory);
- if (length < 0 || (size_t)length >= sizeof(path)) {
- fprintf(stderr, "Model path is too long\n");
- return -1;
- }
- yyjson_read_err error;
- d->config = yyjson_read_file(path, 0, NULL, &error);
- if (!d->config) {
- fprintf(stderr, "Cannot read %s: %s\n", path, error.msg);
- return -1;
- }
- yyjson_val *root = yyjson_doc_get_root(d->config);
- if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
- !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
- "XLMRobertaForSequenceClassification")) {
- fprintf(stderr, "Expected an XLMRobertaForSequenceClassification checkpoint\n");
- return -1;
- }
- yyjson_val *labels = yyjson_obj_get(root, "id2label");
- if (yyjson_obj_size(labels) != NUM_LABELS) {
- fprintf(stderr, "Expected %d language labels in config.json\n", NUM_LABELS);
- return -1;
- }
- for (int i = 0; i < NUM_LABELS; i++) {
- char key[16];
- snprintf(key, sizeof(key), "%d", i);
- d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
- if (!d->labels[i] || !d->labels[i][0]) {
- fprintf(stderr, "Missing language label %d\n", i);
- return -1;
- }
- }
-
- d->model = langdet_model_load(directory);
- if (!d->model)
- return -1;
- const langdet_config_t *cfg = langdet_model_config(d->model);
- if (cfg->hidden_size != HIDDEN_SIZE ||
- cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) {
- fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n");
- return -1;
- }
- d->dense_weight = load_tensor(d, "classifier.dense.weight",
- (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2);
- d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1);
- d->out_weight = load_tensor(d, "classifier.out_proj.weight",
- (int64_t[]){NUM_LABELS, HIDDEN_SIZE}, 2);
- d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){NUM_LABELS}, 1);
- if (!d->dense_weight || !d->dense_bias || !d->out_weight || !d->out_bias)
- return -1;
-
- char message[256] = {0};
- d->tokenizer = langdet_tok_open(directory, message, sizeof(message));
- if (!d->tokenizer) {
- fprintf(stderr, "Tokenizer: %s\n", message);
- return -1;
- }
- d->workspace = langdet_workspace_new(d->model);
- if (!d->workspace) {
- fprintf(stderr, "Cannot allocate inference workspace\n");
- return -1;
- }
- return 0;
-}
-
-static int classify(detector_t *d, const char *text, int all_scores) {
+static int print_prediction(langdet_t *detector, const char *text, int all_scores) {
double start = now_ms();
- int count = 0;
- int *ids = langdet_tokenize(d->tokenizer, text, &count);
- if (!ids || count < 2) {
- fprintf(stderr, "Tokenization failed\n");
- free(ids);
+ langdet_result_t result;
+ char error[256];
+ if (langdet_predict(detector, text, &result, error, sizeof(error)) != 0) {
+ fprintf(stderr, "%s\n", error);
return -1;
}
- int truncated = count > MAX_TOKENS;
- if (truncated) {
- ids[MAX_TOKENS - 1] = ids[count - 1];
- count = MAX_TOKENS;
- }
- float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS];
- int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
- free(ids);
- if (result != 0) {
- fprintf(stderr, "Encoder inference failed\n");
- return -1;
- }
- linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
- for (int i = 0; i < HIDDEN_SIZE; i++)
- dense[i] = tanhf(dense[i]);
- linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, NUM_LABELS);
- softmax(scores, 1, NUM_LABELS);
- int best = 0;
- for (int i = 0; i < NUM_LABELS; i++) {
- if (!isfinite(scores[i])) {
- fprintf(stderr, "Classifier produced a non-finite score\n");
- return -1;
- }
- if (scores[i] > scores[best])
- best = i;
- }
double elapsed = now_ms() - start;
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
- if (!doc)
+ if (!doc) {
+ fprintf(stderr, "Cannot allocate JSON output\n");
return -1;
+ }
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
int ok = root &&
- yyjson_mut_obj_add_str(doc, root, "label", d->labels[best]) &&
- yyjson_mut_obj_add_real(doc, root, "score", scores[best]) &&
- yyjson_mut_obj_add_int(doc, root, "tokens", count) &&
- yyjson_mut_obj_add_bool(doc, root, "truncated", truncated) &&
+ yyjson_mut_obj_add_str(doc, root, "label", result.label) &&
+ yyjson_mut_obj_add_real(doc, root, "score", result.score) &&
+ yyjson_mut_obj_add_int(doc, root, "tokens", result.tokens) &&
+ yyjson_mut_obj_add_bool(doc, root, "truncated", result.truncated) &&
yyjson_mut_obj_add_real(doc, root, "ms", elapsed);
if (all_scores && ok) {
yyjson_mut_val *all = yyjson_mut_obj(doc);
ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all);
- for (int i = 0; i < NUM_LABELS && ok; i++)
- ok = yyjson_mut_obj_add_real(doc, all, d->labels[i], scores[i]);
+ for (int i = 0; i < LANGDET_NUM_LABELS && ok; i++)
+ ok = yyjson_mut_obj_add_real(doc, all, result.labels[i], result.scores[i]);
}
char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL;
int written = json ? puts(json) : EOF;
@@ -199,9 +70,7 @@ static void usage(FILE *out, const char *program) {
int main(int argc, char **argv) {
const char *directory = "model";
- int threads = tp_get_num_cpus();
- if (threads > 16)
- threads = 16;
+ int threads = 0;
int all_scores = 0;
const struct option options[] = {
{"all", no_argument, NULL, 'a'},
@@ -228,16 +97,16 @@ int main(int argc, char **argv) {
default: usage(stderr, argv[0]); return 1;
}
}
- detector_t detector = {0};
- if (detector_load(&detector, directory) != 0) {
- detector_free(&detector);
+ char error[256];
+ langdet_t *detector = langdet_open(directory, threads, error, sizeof(error));
+ if (!detector) {
+ fprintf(stderr, "%s\n", error);
return 1;
}
- tp_set_threads(threads);
int status = 0;
if (optind < argc) {
for (int i = optind; i < argc && status == 0; i++)
- status = classify(&detector, argv[i], all_scores);
+ status = print_prediction(detector, argv[i], all_scores);
} else {
char *line = NULL;
size_t capacity = 0;
@@ -250,7 +119,7 @@ int main(int argc, char **argv) {
}
while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r'))
line[--length] = '\0';
- status = classify(&detector, line, all_scores);
+ status = print_prediction(detector, line, all_scores);
if (status != 0)
break;
}
@@ -260,7 +129,6 @@ int main(int argc, char **argv) {
}
free(line);
}
- tp_set_threads(1);
- detector_free(&detector);
+ langdet_free(detector);
return status == 0 ? 0 : 1;
}