#ifndef LANGDET_ENCODER_H #define LANGDET_ENCODER_H #include #define LANGDET_MAX_LAYERS 64 typedef struct { int hidden_size; int n_layers; int n_heads; int head_dim; int intermediate_size; int vocab_size; int max_position_embeddings; int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ int type_vocab_size; float layer_norm_eps; } langdet_config_t; typedef struct langdet_model langdet_model_t; typedef struct langdet_workspace langdet_workspace_t; typedef struct langdet_tok langdet_tok_t; /* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ langdet_model_t *langdet_model_load(const char *model_dir); void langdet_model_free(langdet_model_t *model); const langdet_config_t *langdet_model_config(const langdet_model_t *model); /* A workspace belongs to one model and may serve only one inference call at a time. */ langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); void langdet_workspace_free(langdet_workspace_t *ws); /* Write the first token's final hidden state to out_embedding[hidden_size]. * Token IDs must include and , without padding. Returns 0 or -1 on error. */ int langdet_model_encode_into(const langdet_model_t *model, langdet_workspace_t *ws, const int *token_ids, int n_tokens, float *out_embedding); langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); void langdet_tok_free(langdet_tok_t *t); /* Return malloc-owned IDs wrapped with and , or NULL on error. * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); #endif /* LANGDET_ENCODER_H */