aboutsummaryrefslogtreecommitdiff
path: root/langdetect.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:54:13 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:54:13 +0200
commit91871be6ece9317c5d49353f6474583e7e28168d (patch)
tree8cabf2546322a9e1b724bf8413f32de54378824f /langdetect.c
parent35ab9c12ba4d396e6cba99d6ea947e26becedad4 (diff)
downloadlangdet-91871be6ece9317c5d49353f6474583e7e28168d.tar.gz
langdet-91871be6ece9317c5d49353f6474583e7e28168d.zip
Add native CPU language detection CLI
Diffstat (limited to 'langdetect.c')
-rw-r--r--langdetect.c269
1 files changed, 269 insertions, 0 deletions
diff --git a/langdetect.c b/langdetect.c
new file mode 100644
index 0000000..5911363
--- /dev/null
+++ b/langdetect.c
@@ -0,0 +1,269 @@
+#include "model_internal.h"
+#include "kernels.h"
+#include "threadpool.h"
+
+#include <yyjson.h>
+
+#include <errno.h>
+#include <getopt.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#define HIDDEN_SIZE 768
+#define NUM_LABELS 20
+#define MAX_TOKENS 512
+
+typedef struct {
+ ffwd_model_t *model;
+ ffwd_workspace_t *workspace;
+ ffwd_tok_t *tokenizer;
+ yyjson_doc *config;
+ const char *labels[NUM_LABELS];
+ const float *dense_weight, *dense_bias;
+ const float *out_weight, *out_bias;
+} detector_t;
+
+static double now_ms(void) {
+ struct timespec t;
+ clock_gettime(CLOCK_MONOTONIC, &t);
+ return t.tv_sec * 1000.0 + t.tv_nsec / 1e6;
+}
+
+static const float *load_tensor(detector_t *d, const char *name, const int64_t *shape, int ndim) {
+ safetensors_file_t *file = NULL;
+ const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
+ if (!tensor) {
+ fprintf(stderr, "Missing classifier tensor: %s\n", name);
+ return NULL;
+ }
+ if (!tensor_has_supported_shape(file, tensor, name, shape, ndim, 0))
+ return NULL;
+ /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */
+ return safetensors_data(file, tensor);
+}
+
+static void detector_free(detector_t *d) {
+ ffwd_tok_free(d->tokenizer);
+ ffwd_workspace_free(d->workspace);
+ ffwd_model_free(d->model);
+ yyjson_doc_free(d->config);
+}
+
+static int detector_load(detector_t *d, const char *directory) {
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/config.json", directory);
+ if (length < 0 || (size_t)length >= sizeof(path)) {
+ fprintf(stderr, "Model path is too long\n");
+ return -1;
+ }
+ yyjson_read_err error;
+ d->config = yyjson_read_file(path, 0, NULL, &error);
+ if (!d->config) {
+ fprintf(stderr, "Cannot read %s: %s\n", path, error.msg);
+ return -1;
+ }
+ yyjson_val *root = yyjson_doc_get_root(d->config);
+ if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
+ !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0),
+ "XLMRobertaForSequenceClassification")) {
+ fprintf(stderr, "Expected an XLMRobertaForSequenceClassification checkpoint\n");
+ return -1;
+ }
+ yyjson_val *labels = yyjson_obj_get(root, "id2label");
+ if (yyjson_obj_size(labels) != NUM_LABELS) {
+ fprintf(stderr, "Expected %d language labels in config.json\n", NUM_LABELS);
+ return -1;
+ }
+ for (int i = 0; i < NUM_LABELS; i++) {
+ char key[16];
+ snprintf(key, sizeof(key), "%d", i);
+ d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key));
+ if (!d->labels[i] || !d->labels[i][0]) {
+ fprintf(stderr, "Missing language label %d\n", i);
+ return -1;
+ }
+ }
+
+ d->model = ffwd_model_load(directory);
+ if (!d->model)
+ return -1;
+ ffwd_config_t *cfg = &d->model->config;
+ if (cfg->family != FFWD_FAMILY_BERT || cfg->hidden_size != HIDDEN_SIZE ||
+ cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) {
+ fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n");
+ return -1;
+ }
+ /* The classifier consumes the unnormalized final state of <s>, not a sentence mean. */
+ cfg->pooling_mode = FFWD_POOL_CLS;
+ cfg->normalize_embeddings = 0;
+ d->dense_weight = load_tensor(d, "classifier.dense.weight",
+ (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2);
+ d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1);
+ d->out_weight = load_tensor(d, "classifier.out_proj.weight",
+ (int64_t[]){NUM_LABELS, HIDDEN_SIZE}, 2);
+ d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){NUM_LABELS}, 1);
+ if (!d->dense_weight || !d->dense_bias || !d->out_weight || !d->out_bias)
+ return -1;
+
+ char message[256] = {0};
+ d->tokenizer = ffwd_tok_open(directory, 0, message, sizeof(message));
+ if (!d->tokenizer) {
+ fprintf(stderr, "Tokenizer: %s\n", message);
+ return -1;
+ }
+ d->workspace = ffwd_workspace_new(d->model);
+ if (!d->workspace) {
+ fprintf(stderr, "Cannot allocate inference workspace\n");
+ return -1;
+ }
+ return 0;
+}
+
+static int classify(detector_t *d, const char *text, int all_scores) {
+ double start = now_ms();
+ int count = 0;
+ int *ids = ffwd_tokenize(d->tokenizer, text, NULL, &count);
+ if (!ids || count < 2) {
+ fprintf(stderr, "Tokenization failed\n");
+ free(ids);
+ return -1;
+ }
+ int truncated = count > MAX_TOKENS;
+ if (truncated) {
+ ids[MAX_TOKENS - 1] = ids[count - 1];
+ count = MAX_TOKENS;
+ }
+ float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS];
+ int result = ffwd_model_encode_into(d->model, d->workspace, ids, count, hidden);
+ free(ids);
+ if (result != 0) {
+ fprintf(stderr, "Encoder inference failed\n");
+ return -1;
+ }
+ linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE);
+ for (int i = 0; i < HIDDEN_SIZE; i++)
+ dense[i] = tanhf(dense[i]);
+ linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, NUM_LABELS);
+ softmax(scores, 1, NUM_LABELS);
+ int best = 0;
+ for (int i = 0; i < NUM_LABELS; i++) {
+ if (!isfinite(scores[i])) {
+ fprintf(stderr, "Classifier produced a non-finite score\n");
+ return -1;
+ }
+ if (scores[i] > scores[best])
+ best = i;
+ }
+ double elapsed = now_ms() - start;
+
+ yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
+ if (!doc)
+ return -1;
+ yyjson_mut_val *root = yyjson_mut_obj(doc);
+ yyjson_mut_doc_set_root(doc, root);
+ int ok = root &&
+ yyjson_mut_obj_add_str(doc, root, "label", d->labels[best]) &&
+ yyjson_mut_obj_add_real(doc, root, "score", scores[best]) &&
+ yyjson_mut_obj_add_int(doc, root, "tokens", count) &&
+ yyjson_mut_obj_add_bool(doc, root, "truncated", truncated) &&
+ yyjson_mut_obj_add_real(doc, root, "ms", elapsed);
+ if (all_scores && ok) {
+ yyjson_mut_val *all = yyjson_mut_obj(doc);
+ ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all);
+ for (int i = 0; i < NUM_LABELS && ok; i++)
+ ok = yyjson_mut_obj_add_real(doc, all, d->labels[i], scores[i]);
+ }
+ char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL;
+ int written = json ? puts(json) : EOF;
+ free(json);
+ yyjson_mut_doc_free(doc);
+ if (written == EOF || fflush(stdout) == EOF) {
+ fprintf(stderr, "Cannot write classification result\n");
+ return -1;
+ }
+ return 0;
+}
+
+static void usage(FILE *out, const char *program) {
+ fprintf(out,
+ "Usage: %s [-d MODEL_DIR] [-t THREADS] [--all] [--] [TEXT ...]\n"
+ "Without TEXT arguments, read one UTF-8 text per line from stdin.\n"
+ "Write one JSON result per text, in input order.\n"
+ " -d DIR Model directory (default: ./model)\n"
+ " -t N CPU kernel threads, 1-16 (default: detected CPU count, capped at 16)\n"
+ " --all Include scores for all 20 languages\n"
+ " -h Show help\n"
+ "Inputs longer than 512 tokens are truncated, retaining the final </s>.\n",
+ program);
+}
+
+int main(int argc, char **argv) {
+ const char *directory = "model";
+ int threads = tp_get_num_cpus();
+ if (threads > 16)
+ threads = 16;
+ int all_scores = 0;
+ const struct option options[] = {
+ {"all", no_argument, NULL, 'a'},
+ {"help", no_argument, NULL, 'h'},
+ {NULL, 0, NULL, 0},
+ };
+ int option;
+ while ((option = getopt_long(argc, argv, "d:t:ah", options, NULL)) != -1) {
+ switch (option) {
+ case 'd': directory = optarg; break;
+ case 'a': all_scores = 1; break;
+ case 'h': usage(stdout, argv[0]); return 0;
+ case 't': {
+ char *end;
+ errno = 0;
+ long n = strtol(optarg, &end, 10);
+ if (errno || *end || n < 1 || n > 16) {
+ fprintf(stderr, "Thread count must be an integer between 1 and 16\n");
+ return 1;
+ }
+ threads = (int)n;
+ break;
+ }
+ default: usage(stderr, argv[0]); return 1;
+ }
+ }
+ detector_t detector = {0};
+ if (detector_load(&detector, directory) != 0) {
+ detector_free(&detector);
+ return 1;
+ }
+ tp_set_threads(threads);
+ int status = 0;
+ if (optind < argc) {
+ for (int i = optind; i < argc && status == 0; i++)
+ status = classify(&detector, argv[i], all_scores);
+ } else {
+ char *line = NULL;
+ size_t capacity = 0;
+ ssize_t length;
+ while ((length = getline(&line, &capacity, stdin)) >= 0) {
+ if (memchr(line, '\0', (size_t)length)) {
+ fprintf(stderr, "Input must not contain NUL bytes\n");
+ status = -1;
+ break;
+ }
+ while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r'))
+ line[--length] = '\0';
+ status = classify(&detector, line, all_scores);
+ if (status != 0)
+ break;
+ }
+ if (ferror(stdin) || (!feof(stdin) && status == 0)) {
+ fprintf(stderr, "Cannot read input\n");
+ status = -1;
+ }
+ free(line);
+ }
+ tp_set_threads(1);
+ detector_free(&detector);
+ return status == 0 ? 0 : 1;
+}