blob: 7d954df25663fcf4fc22df3cb6c9730d1d936fa7 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
|
#ifndef LANGDET_ENCODER_H
#define LANGDET_ENCODER_H
#include <stddef.h>
#define LANGDET_MAX_LAYERS 64
typedef struct {
int hidden_size;
int n_layers;
int n_heads;
int head_dim;
int intermediate_size;
int vocab_size;
int max_position_embeddings;
int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
int type_vocab_size;
float layer_norm_eps;
} langdet_config_t;
typedef struct langdet_model langdet_model_t;
typedef struct langdet_workspace langdet_workspace_t;
typedef struct langdet_tok langdet_tok_t;
/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
langdet_model_t *langdet_model_load(const char *model_dir);
void langdet_model_free(langdet_model_t *model);
const langdet_config_t *langdet_model_config(const langdet_model_t *model);
/* A workspace belongs to one model and may serve only one inference call at a time. */
langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
void langdet_workspace_free(langdet_workspace_t *ws);
/* Write the first token's final hidden state to out_embedding[hidden_size].
* Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
int langdet_model_encode_into(const langdet_model_t *model,
langdet_workspace_t *ws,
const int *token_ids,
int n_tokens,
float *out_embedding);
langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
void langdet_tok_free(langdet_tok_t *t);
/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
* The tokenizer owns reusable scratch and is not safe for concurrent calls. */
int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
#endif /* LANGDET_ENCODER_H */
|