aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/encoder.h
blob: 7d954df25663fcf4fc22df3cb6c9730d1d936fa7 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#ifndef LANGDET_ENCODER_H
#define LANGDET_ENCODER_H

#include <stddef.h>

#define LANGDET_MAX_LAYERS 64

typedef struct {
    int hidden_size;
    int n_layers;
    int n_heads;
    int head_dim;
    int intermediate_size;
    int vocab_size;
    int max_position_embeddings;
    int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
    int type_vocab_size;
    float layer_norm_eps;
} langdet_config_t;

typedef struct langdet_model langdet_model_t;
typedef struct langdet_workspace langdet_workspace_t;
typedef struct langdet_tok langdet_tok_t;

/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
langdet_model_t *langdet_model_load(const char *model_dir);
void langdet_model_free(langdet_model_t *model);
const langdet_config_t *langdet_model_config(const langdet_model_t *model);

/* A workspace belongs to one model and may serve only one inference call at a time. */
langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
void langdet_workspace_free(langdet_workspace_t *ws);

/* Write the first token's final hidden state to out_embedding[hidden_size].
 * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
int langdet_model_encode_into(const langdet_model_t *model,
                              langdet_workspace_t *ws,
                              const int *token_ids,
                              int n_tokens,
                              float *out_embedding);

langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
void langdet_tok_free(langdet_tok_t *t);

/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
 * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);

#endif /* LANGDET_ENCODER_H */