aboutsummaryrefslogtreecommitdiff
path: root/langdetect.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
commitcf7d8169ec12cb3adf90d4ff78b3257e74985302 (patch)
treef666589acfd68e76bbde4e74eead44ae7819d0bb /langdetect.c
parent91871be6ece9317c5d49353f6474583e7e28168d (diff)
downloadlangdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.tar.gz
langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.zip
Remove unused model families and inference paths
Diffstat (limited to 'langdetect.c')
-rw-r--r--langdetect.c13
1 files changed, 5 insertions, 8 deletions
diff --git a/langdetect.c b/langdetect.c
index 5911363..003692d 100644
--- a/langdetect.c
+++ b/langdetect.c
@@ -39,7 +39,7 @@ static const float *load_tensor(detector_t *d, const char *name, const int64_t *
fprintf(stderr, "Missing classifier tensor: %s\n", name);
return NULL;
}
- if (!tensor_has_supported_shape(file, tensor, name, shape, ndim, 0))
+ if (!tensor_has_supported_shape(file, tensor, name, shape, ndim))
return NULL;
/* Classifier pointers borrow the model's mapping and remain valid until model teardown. */
return safetensors_data(file, tensor);
@@ -90,15 +90,12 @@ static int detector_load(detector_t *d, const char *directory) {
d->model = ffwd_model_load(directory);
if (!d->model)
return -1;
- ffwd_config_t *cfg = &d->model->config;
- if (cfg->family != FFWD_FAMILY_BERT || cfg->hidden_size != HIDDEN_SIZE ||
+ const ffwd_config_t *cfg = ffwd_model_config(d->model);
+ if (cfg->hidden_size != HIDDEN_SIZE ||
cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) {
fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n");
return -1;
}
- /* The classifier consumes the unnormalized final state of <s>, not a sentence mean. */
- cfg->pooling_mode = FFWD_POOL_CLS;
- cfg->normalize_embeddings = 0;
d->dense_weight = load_tensor(d, "classifier.dense.weight",
(int64_t[]){HIDDEN_SIZE, HIDDEN_SIZE}, 2);
d->dense_bias = load_tensor(d, "classifier.dense.bias", (int64_t[]){HIDDEN_SIZE}, 1);
@@ -109,7 +106,7 @@ static int detector_load(detector_t *d, const char *directory) {
return -1;
char message[256] = {0};
- d->tokenizer = ffwd_tok_open(directory, 0, message, sizeof(message));
+ d->tokenizer = ffwd_tok_open(directory, message, sizeof(message));
if (!d->tokenizer) {
fprintf(stderr, "Tokenizer: %s\n", message);
return -1;
@@ -125,7 +122,7 @@ static int detector_load(detector_t *d, const char *directory) {
static int classify(detector_t *d, const char *text, int all_scores) {
double start = now_ms();
int count = 0;
- int *ids = ffwd_tokenize(d->tokenizer, text, NULL, &count);
+ int *ids = ffwd_tokenize(d->tokenizer, text, &count);
if (!ids || count < 2) {
fprintf(stderr, "Tokenization failed\n");
free(ids);