diff options
Diffstat (limited to 'liblangdet/encoder.h')
| -rw-r--r-- | liblangdet/encoder.h | 49 |
1 files changed, 49 insertions, 0 deletions
diff --git a/liblangdet/encoder.h b/liblangdet/encoder.h new file mode 100644 index 0000000..7d954df --- /dev/null +++ b/liblangdet/encoder.h @@ -0,0 +1,49 @@ +#ifndef LANGDET_ENCODER_H +#define LANGDET_ENCODER_H + +#include <stddef.h> + +#define LANGDET_MAX_LAYERS 64 + +typedef struct { + int hidden_size; + int n_layers; + int n_heads; + int head_dim; + int intermediate_size; + int vocab_size; + int max_position_embeddings; + int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ + int type_vocab_size; + float layer_norm_eps; +} langdet_config_t; + +typedef struct langdet_model langdet_model_t; +typedef struct langdet_workspace langdet_workspace_t; +typedef struct langdet_tok langdet_tok_t; + +/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ +langdet_model_t *langdet_model_load(const char *model_dir); +void langdet_model_free(langdet_model_t *model); +const langdet_config_t *langdet_model_config(const langdet_model_t *model); + +/* A workspace belongs to one model and may serve only one inference call at a time. */ +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); +void langdet_workspace_free(langdet_workspace_t *ws); + +/* Write the first token's final hidden state to out_embedding[hidden_size]. + * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, + const int *token_ids, + int n_tokens, + float *out_embedding); + +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); +void langdet_tok_free(langdet_tok_t *t); + +/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. + * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); + +#endif /* LANGDET_ENCODER_H */ |
