diff options
Diffstat (limited to 'langdetect.c')
| -rw-r--r-- | langdetect.c | 178 |
1 files changed, 23 insertions, 155 deletions
diff --git a/langdetect.c b/langdetect.c index 6418d2d..76d74ab 100644 --- a/langdetect.c +++ b/langdetect.c @@ -1,177 +1,48 @@ -#include "model_internal.h" -#include "kernels.h" -#include "threadpool.h" +#include "langdet.h" #include <yyjson.h> #include <errno.h> #include <getopt.h> -#include <math.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <time.h> -#define HIDDEN_SIZE 768 -#define NUM_LABELS 20 -#define MAX_TOKENS 512 - -typedef struct { - langdet_model_t *model; - langdet_workspace_t *workspace; - langdet_tok_t *tokenizer; - yyjson_doc *config; - const char *labels[NUM_LABELS]; - const float *dense_weight, *dense_bias; - const float *out_weight, *out_bias; -} detector_t; - static double now_ms(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec * 1000.0 + t.tv_nsec / 1e6; } -static const float *load_tensor(detector_t *d, const char *name, const int64_t *shape, int ndim) { - safetensors_file_t *file = NULL; - const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file); - if (!tensor) { - fprintf(stderr, "Missing classifier tensor: %s\n", name); - return NULL; - } - if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) - return NULL; - /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */ - return safetensors_data(file, tensor); -} - -static void detector_free(detector_t *d) { - langdet_tok_free(d->tokenizer); - langdet_workspace_free(d->workspace); - langdet_model_free(d->model); - yyjson_doc_free(d->config); -} - -static int detector_load(detector_t *d, const char *directory) { - char path[1024]; - int length = snprintf(path, sizeof(path), "%s/config.json", directory); - if (length < 0 || (size_t)length >= sizeof(path)) { - fprintf(stderr, "Model path is too long\n"); - return -1; - } - yyjson_read_err error; - d->config = yyjson_read_file(path, 0, NULL, &error); - if (!d->config) { - fprintf(stderr, "Cannot read %s: %s\n", path, error.msg); - return -1; - } - yyjson_val *root = yyjson_doc_get_root(d->config); - if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || - !yyjson_equals_str(yyjson_arr_get(yyjson_obj_get(root, "architectures"), 0), - "XLMRobertaForSequenceClassification")) { - fprintf(stderr, "Expected an XLMRobertaForSequenceClassification checkpoint\n"); - return -1; - } - yyjson_val *labels = yyjson_obj_get(root, "id2label"); - if (yyjson_obj_size(labels) != NUM_LABELS) { - fprintf(stderr, "Expected %d language labels in config.json\n", NUM_LABELS); - return -1; - } - for (int i = 0; i < NUM_LABELS; i++) { - char key[16]; - snprintf(key, sizeof(key), "%d", i); - d->labels[i] = yyjson_get_str(yyjson_obj_get(labels, key)); - if (!d->labels[i] || !d->labels[i][0]) { - fprintf(stderr, "Missing language label %d\n", i); - return -1; - } - } - - d->model = langdet_model_load(directory); - if (!d->model) - return -1; - const langdet_config_t *cfg = langdet_model_config(d->model); - if (cfg->hidden_size != HIDDEN_SIZE || - cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) { - fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n"); - return -1; - } - d->dense_weight = load_tensor(d, "classifier.dense.weight", - (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2); - d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1); - d->out_weight = load_tensor(d, "classifier.out_proj.weight", - (int64_t[]){NUM_LABELS, HIDDEN_SIZE}, 2); - d->out_bias = load_tensor(d, "classifier.out_proj.bias", (int64_t[]){NUM_LABELS}, 1); - if (!d->dense_weight || !d->dense_bias || !d->out_weight || !d->out_bias) - return -1; - - char message[256] = {0}; - d->tokenizer = langdet_tok_open(directory, message, sizeof(message)); - if (!d->tokenizer) { - fprintf(stderr, "Tokenizer: %s\n", message); - return -1; - } - d->workspace = langdet_workspace_new(d->model); - if (!d->workspace) { - fprintf(stderr, "Cannot allocate inference workspace\n"); - return -1; - } - return 0; -} - -static int classify(detector_t *d, const char *text, int all_scores) { +static int print_prediction(langdet_t *detector, const char *text, int all_scores) { double start = now_ms(); - int count = 0; - int *ids = langdet_tokenize(d->tokenizer, text, &count); - if (!ids || count < 2) { - fprintf(stderr, "Tokenization failed\n"); - free(ids); + langdet_result_t result; + char error[256]; + if (langdet_predict(detector, text, &result, error, sizeof(error)) != 0) { + fprintf(stderr, "%s\n", error); return -1; } - int truncated = count > MAX_TOKENS; - if (truncated) { - ids[MAX_TOKENS - 1] = ids[count - 1]; - count = MAX_TOKENS; - } - float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS]; - int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); - free(ids); - if (result != 0) { - fprintf(stderr, "Encoder inference failed\n"); - return -1; - } - linear(dense, hidden, d->dense_weight, d->dense_bias, 1, HIDDEN_SIZE, HIDDEN_SIZE); - for (int i = 0; i < HIDDEN_SIZE; i++) - dense[i] = tanhf(dense[i]); - linear(scores, dense, d->out_weight, d->out_bias, 1, HIDDEN_SIZE, NUM_LABELS); - softmax(scores, 1, NUM_LABELS); - int best = 0; - for (int i = 0; i < NUM_LABELS; i++) { - if (!isfinite(scores[i])) { - fprintf(stderr, "Classifier produced a non-finite score\n"); - return -1; - } - if (scores[i] > scores[best]) - best = i; - } double elapsed = now_ms() - start; yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - if (!doc) + if (!doc) { + fprintf(stderr, "Cannot allocate JSON output\n"); return -1; + } yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); int ok = root && - yyjson_mut_obj_add_str(doc, root, "label", d->labels[best]) && - yyjson_mut_obj_add_real(doc, root, "score", scores[best]) && - yyjson_mut_obj_add_int(doc, root, "tokens", count) && - yyjson_mut_obj_add_bool(doc, root, "truncated", truncated) && + yyjson_mut_obj_add_str(doc, root, "label", result.label) && + yyjson_mut_obj_add_real(doc, root, "score", result.score) && + yyjson_mut_obj_add_int(doc, root, "tokens", result.tokens) && + yyjson_mut_obj_add_bool(doc, root, "truncated", result.truncated) && yyjson_mut_obj_add_real(doc, root, "ms", elapsed); if (all_scores && ok) { yyjson_mut_val *all = yyjson_mut_obj(doc); ok = all && yyjson_mut_obj_add_val(doc, root, "scores", all); - for (int i = 0; i < NUM_LABELS && ok; i++) - ok = yyjson_mut_obj_add_real(doc, all, d->labels[i], scores[i]); + for (int i = 0; i < LANGDET_NUM_LABELS && ok; i++) + ok = yyjson_mut_obj_add_real(doc, all, result.labels[i], result.scores[i]); } char *json = ok ? yyjson_mut_write(doc, 0, NULL) : NULL; int written = json ? puts(json) : EOF; @@ -199,9 +70,7 @@ static void usage(FILE *out, const char *program) { int main(int argc, char **argv) { const char *directory = "model"; - int threads = tp_get_num_cpus(); - if (threads > 16) - threads = 16; + int threads = 0; int all_scores = 0; const struct option options[] = { {"all", no_argument, NULL, 'a'}, @@ -228,16 +97,16 @@ int main(int argc, char **argv) { default: usage(stderr, argv[0]); return 1; } } - detector_t detector = {0}; - if (detector_load(&detector, directory) != 0) { - detector_free(&detector); + char error[256]; + langdet_t *detector = langdet_open(directory, threads, error, sizeof(error)); + if (!detector) { + fprintf(stderr, "%s\n", error); return 1; } - tp_set_threads(threads); int status = 0; if (optind < argc) { for (int i = optind; i < argc && status == 0; i++) - status = classify(&detector, argv[i], all_scores); + status = print_prediction(detector, argv[i], all_scores); } else { char *line = NULL; size_t capacity = 0; @@ -250,7 +119,7 @@ int main(int argc, char **argv) { } while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r')) line[--length] = '\0'; - status = classify(&detector, line, all_scores); + status = print_prediction(detector, line, all_scores); if (status != 0) break; } @@ -260,7 +129,6 @@ int main(int argc, char **argv) { } free(line); } - tp_set_threads(1); - detector_free(&detector); + langdet_free(detector); return status == 0 ? 0 : 1; } |
