diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:40:12 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:40:12 +0200 |
| commit | cb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch) | |
| tree | 85edb98bacb5ce8da99245343f22fe58e594229b /liblangdet/langdet.h | |
| parent | 53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff) | |
| download | langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip | |
Expose complete language prediction API and simplify CLI
Diffstat (limited to 'liblangdet/langdet.h')
| -rw-r--r-- | liblangdet/langdet.h | 71 |
1 files changed, 36 insertions, 35 deletions
diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h index 9c24264..22e0b64 100644 --- a/liblangdet/langdet.h +++ b/liblangdet/langdet.h @@ -3,47 +3,48 @@ #include <stddef.h> -#define LANGDET_MAX_LAYERS 64 +#ifdef __cplusplus +extern "C" { +#endif -typedef struct { - int hidden_size; - int n_layers; - int n_heads; - int head_dim; - int intermediate_size; - int vocab_size; - int max_position_embeddings; - int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ - int type_vocab_size; - float layer_norm_eps; -} langdet_config_t; +#define LANGDET_NUM_LABELS 20 +#define LANGDET_MAX_TOKENS 512 + +typedef struct langdet langdet_t; -typedef struct langdet_model langdet_model_t; -typedef struct langdet_workspace langdet_workspace_t; -typedef struct langdet_tok langdet_tok_t; +typedef struct { + const char *label; + float score; + int tokens; + int truncated; + /* Label pointers belong to the detector and remain valid until langdet_free(). + * Scores are stored in this result, in the checkpoint's id2label order. */ + const char *labels[LANGDET_NUM_LABELS]; + float scores[LANGDET_NUM_LABELS]; +} langdet_result_t; -/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ -langdet_model_t *langdet_model_load(const char *model_dir); -void langdet_model_free(langdet_model_t *model); -const langdet_config_t *langdet_model_config(const langdet_model_t *model); +/* Library calls must not overlap across application threads, even for different + * handles. Internal CPU workers cooperate on one prediction at a time. */ -/* A workspace belongs to one model and may serve only one inference call at a time. */ -langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); -void langdet_workspace_free(langdet_workspace_t *ws); +/* Load the model, tokenizer, classifier, and labels from model_dir. + * n_threads is 1..16, or 0 to detect the CPU count and cap it at 16. + * Return NULL on failure. An optional err[errlen] receives a NUL-terminated + * message on failure and is cleared on success. Model-file readers may also + * write diagnostics to stderr. */ +langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen); -/* Write the first token's final hidden state to out_embedding[hidden_size]. - * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ -int langdet_model_encode_into(const langdet_model_t *model, - langdet_workspace_t *ws, - const int *token_ids, - int n_tokens, - float *out_embedding); +/* Classify NUL-terminated UTF-8 text. Truncate to 512 tokens, retaining </s>. + * Empty text is accepted but does not yield a meaningful language prediction. + * Return 0 on success, or -1 with a zeroed result on failure. The optional + * error buffer follows the same contract as langdet_open(). */ +int langdet_predict(langdet_t *detector, const char *text, langdet_result_t *result, + char *err, size_t errlen); -langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); -void langdet_tok_free(langdet_tok_t *t); +/* Release the detector and its resources. NULL is accepted. */ +void langdet_free(langdet_t *detector); -/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. - * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ -int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); +#ifdef __cplusplus +} +#endif #endif /* LANGDET_H */ |
