diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:54:13 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:54:13 +0200 |
| commit | 91871be6ece9317c5d49353f6474583e7e28168d (patch) | |
| tree | 8cabf2546322a9e1b724bf8413f32de54378824f /langdetect.c | |
| parent | 35ab9c12ba4d396e6cba99d6ea947e26becedad4 (diff) | |
| download | langdet-91871be6ece9317c5d49353f6474583e7e28168d.tar.gz langdet-91871be6ece9317c5d49353f6474583e7e28168d.zip | |
Add native CPU language detection CLI
Diffstat (limited to 'langdetect.c')
| -rw-r--r-- | langdetect.c | 269 |
1 files changed, 269 insertions, 0 deletions
diff --git a/langdetect.c b/langdetect.c new file mode 100644 index 0000000..5911363 --- /dev/null +++ b/langdetect.c @@ -0,0 +1,269 @@ +#include "model_internal.h" +#include "kernels.h" +#include "threadpool.h" + +#include <yyjson.h> + +#include <errno.h> +#include <getopt.h> +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#define HIDDEN_SIZE 768 +#define NUM_LABELS 20 +#define MAX_TOKENS 512 + +typedef struct { + ffwd_model_t *model; + ffwd_workspace_t *workspace; + ffwd_tok_t *tokenizer; + yyjson_doc *config; + const char *labels[NUM_LABELS]; + const float *dense_weight, *dense_bias; + const float *out_weight, *out_bias; +} detector_t; + +static double now_ms(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return t.tv_sec * 1000.0 + t.tv_nsec / 1e6; +} + +static const float *load_tensor(detector_t *d, const char *name, const int64_t *shape, int ndim) { + safetensors_file_t *file = NULL; + const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file); + if (!tensor) { + fprintf(stderr, "Missing classifier tensor: %s\n", name); + return NULL; + } + if (!tensor_has_supported_shape(file, tensor, name, shape, ndim, 0)) + return NULL; + /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */ + return safetensors_data(file, tensor); +} + +static void detector_free(detector_t *d) { + ffwd_tok_free(d->tokenizer); + ffwd_workspace_free(d->workspace); + ffwd_model_free(d->model); + yyjson_doc_free(d->config); +} + +static int detector_load(detector_t *d, const char *directory) { + char path[1024]; + int length = snprintf(path, sizeof(path), "%s/config.json", directory); + if (length < 0 || (size_t)length >= sizeof(path)) { + fprintf(stderr, "Model path is too long\n"); + return -1; + } + yyjson_read_err error; + d->config = yyjson_read_file(path, 0, NULL, &error); + if (!d->config) { + fprintf(stderr, "Cannot read %s: %s\n", path, error.msg); + return -1; + } + yyjson_val *root = yyjson_doc_get_root(d->config); + if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || + !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0), + "XLMRobertaForSequenceClassification")) { + fprintf(stderr, "Expected an XLMRobertaForSequenceClassification checkpoint\n"); + return -1; + } + yyjson_val *labels = yyjson_obj_get(root, "id2label"); + if (yyjson_obj_size(labels) != NUM_LABELS) { + fprintf(stderr, "Expected %d language labels in config.json\n", NUM_LABELS); + return -1; + } + for (int i = 0; i < NUM_LABELS; i++) { + char key[16]; + snprintf(key, sizeof(key), "%d", i); + d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key)); + if (!d->labels[i] || !d->labels[i][0]) { + fprintf(stderr, "Missing language label %d\n", i); + return -1; + } + } + + d->model = ffwd_model_load(directory); + if (!d->model) + return -1; + ffwd_config_t *cfg = &d->model->config; + if (cfg->family != FFWD_FAMILY_BERT || cfg->hidden_size != HIDDEN_SIZE || + cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) { + fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n"); + return -1; + } + /* The classifier consumes the unnormalized final state of <s>, not a sentence mean. */ + cfg->pooling_mode = FFWD_POOL_CLS; + cfg->normalize_embeddings = 0; + d->dense_weight = load_tensor(d, "classifier.dense.weight", + (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2); + d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1); + d->out_weight = load_tensor(d, "classifier.out_proj.weight", + (int64_t[]){NUM_LABELS, HIDDEN_SIZE}, 2); + d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){NUM_LABELS}, 1); + if (!d->dense_weight || !d->dense_bias || !d->out_weight || !d->out_bias) + return -1; + + char message[256] = {0}; + d->tokenizer = ffwd_tok_open(directory, 0, message, sizeof(message)); + if (!d->tokenizer) { + fprintf(stderr, "Tokenizer: %s\n", message); + return -1; + } + d->workspace = ffwd_workspace_new(d->model); + if (!d->workspace) { + fprintf(stderr, "Cannot allocate inference workspace\n"); + return -1; + } + return 0; +} + +static int classify(detector_t *d, const char *text, int all_scores) { + double start = now_ms(); + int count = 0; + int *ids = ffwd_tokenize(d->tokenizer, text, NULL, &count); + if (!ids || count < 2) { + fprintf(stderr, "Tokenization failed\n"); + free(ids); + return -1; + } + int truncated = count > MAX_TOKENS; + if (truncated) { + ids[MAX_TOKENS - 1] = ids[count - 1]; + count = MAX_TOKENS; + } + float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS]; + int result = ffwd_model_encode_into(d->model, d->workspace, ids, count, hidden); + free(ids); + if (result != 0) { + fprintf(stderr, "Encoder inference failed\n"); + return -1; + } + linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE); + for (int i = 0; i < HIDDEN_SIZE; i++) + dense[i] = tanhf(dense[i]); + linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, NUM_LABELS); + softmax(scores, 1, NUM_LABELS); + int best = 0; + for (int i = 0; i < NUM_LABELS; i++) { + if (!isfinite(scores[i])) { + fprintf(stderr, "Classifier produced a non-finite score\n"); + return -1; + } + if (scores[i] > scores[best]) + best = i; + } + double elapsed = now_ms() - start; + + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + if (!doc) + return -1; + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + int ok = root && + yyjson_mut_obj_add_str(doc, root, "label", d->labels[best]) && + yyjson_mut_obj_add_real(doc, root, "score", scores[best]) && + yyjson_mut_obj_add_int(doc, root, "tokens", count) && + yyjson_mut_obj_add_bool(doc, root, "truncated", truncated) && + yyjson_mut_obj_add_real(doc, root, "ms", elapsed); + if (all_scores && ok) { + yyjson_mut_val *all = yyjson_mut_obj(doc); + ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all); + for (int i = 0; i < NUM_LABELS && ok; i++) + ok = yyjson_mut_obj_add_real(doc, all, d->labels[i], scores[i]); + } + char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL; + int written = json ? puts(json) : EOF; + free(json); + yyjson_mut_doc_free(doc); + if (written == EOF || fflush(stdout) == EOF) { + fprintf(stderr, "Cannot write classification result\n"); + return -1; + } + return 0; +} + +static void usage(FILE *out, const char *program) { + fprintf(out, + "Usage: %s [-d MODEL_DIR] [-t THREADS] [--all] [--] [TEXT ...]\n" + "Without TEXT arguments, read one UTF-8 text per line from stdin.\n" + "Write one JSON result per text, in input order.\n" + " -d DIR Model directory (default: ./model)\n" + " -t N CPU kernel threads, 1-16 (default: detected CPU count, capped at 16)\n" + " --all Include scores for all 20 languages\n" + " -h Show help\n" + "Inputs longer than 512 tokens are truncated, retaining the final </s>.\n", + program); +} + +int main(int argc, char **argv) { + const char *directory = "model"; + int threads = tp_get_num_cpus(); + if (threads > 16) + threads = 16; + int all_scores = 0; + const struct option options[] = { + {"all", no_argument, NULL, 'a'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0}, + }; + int option; + while ((option = getopt_long(argc, argv, "d:t:ah", options, NULL)) != -1) { + switch (option) { + case 'd': directory = optarg; break; + case 'a': all_scores = 1; break; + case 'h': usage(stdout, argv[0]); return 0; + case 't': { + char *end; + errno = 0; + long n = strtol(optarg, &end, 10); + if (errno || *end || n < 1 || n > 16) { + fprintf(stderr, "Thread count must be an integer between 1 and 16\n"); + return 1; + } + threads = (int)n; + break; + } + default: usage(stderr, argv[0]); return 1; + } + } + detector_t detector = {0}; + if (detector_load(&detector, directory) != 0) { + detector_free(&detector); + return 1; + } + tp_set_threads(threads); + int status = 0; + if (optind < argc) { + for (int i = optind; i < argc && status == 0; i++) + status = classify(&detector, argv[i], all_scores); + } else { + char *line = NULL; + size_t capacity = 0; + ssize_t length; + while ((length = getline(&line, &capacity, stdin)) >= 0) { + if (memchr(line, '\0', (size_t)length)) { + fprintf(stderr, "Input must not contain NUL bytes\n"); + status = -1; + break; + } + while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r')) + line[--length] = '\0'; + status = classify(&detector, line, all_scores); + if (status != 0) + break; + } + if (ferror(stdin) || (!feof(stdin) && status == 0)) { + fprintf(stderr, "Cannot read input\n"); + status = -1; + } + free(line); + } + tp_set_threads(1); + detector_free(&detector); + return status == 0 ? 0 : 1; +} |
