diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:05:44 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:05:44 +0200 |
| commit | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (patch) | |
| tree | f666589acfd68e76bbde4e74eead44ae7819d0bb /langdetect.c | |
| parent | 91871be6ece9317c5d49353f6474583e7e28168d (diff) | |
| download | langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.tar.gz langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.zip | |
Remove unused model families and inference paths
Diffstat (limited to 'langdetect.c')
| -rw-r--r-- | langdetect.c | 13 |
1 files changed, 5 insertions, 8 deletions
diff --git a/langdetect.c b/langdetect.c index 5911363..003692d 100644 --- a/langdetect.c +++ b/langdetect.c @@ -39,7 +39,7 @@ static const float *load_tensor(detector_t *d, const char *name, const int64_t * fprintf(stderr, "Missing classifier tensor: %s\n", name); return NULL; } - if (!tensor_has_supported_shape(file, tensor, name, shape, ndim, 0)) + if (!tensor_has_supported_shape(file, tensor, name, shape, ndim)) return NULL; /* Classifier pointers borrow the model's mapping and remain valid until model teardown. */ return safetensors_data(file, tensor); @@ -90,15 +90,12 @@ static int detector_load(detector_t *d, const char *directory) { d->model = ffwd_model_load(directory); if (!d->model) return -1; - ffwd_config_t *cfg = &d->model->config; - if (cfg->family != FFWD_FAMILY_BERT || cfg->hidden_size != HIDDEN_SIZE || + const ffwd_config_t *cfg = ffwd_model_config(d->model); + if (cfg->hidden_size != HIDDEN_SIZE || cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) { fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n"); return -1; } - /* The classifier consumes the unnormalized final state of <s>, not a sentence mean. */ - cfg->pooling_mode = FFWD_POOL_CLS; - cfg->normalize_embeddings = 0; d->dense_weight = load_tensor(d, "classifier.dense.weight", (int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2); d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1); @@ -109,7 +106,7 @@ static int detector_load(detector_t *d, const char *directory) { return -1; char message[256] = {0}; - d->tokenizer = ffwd_tok_open(directory, 0, message, sizeof(message)); + d->tokenizer = ffwd_tok_open(directory, message, sizeof(message)); if (!d->tokenizer) { fprintf(stderr, "Tokenizer: %s\n", message); return -1; @@ -125,7 +122,7 @@ static int detector_load(detector_t *d, const char *directory) { static int classify(detector_t *d, const char *text, int all_scores) { double start = now_ms(); int count = 0; - int *ids = ffwd_tokenize(d->tokenizer, text, NULL, &count); + int *ids = ffwd_tokenize(d->tokenizer, text, &count); if (!ids || count < 2) { fprintf(stderr, "Tokenization failed\n"); free(ids); |
