aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/langdet.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
commitcb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch)
tree85edb98bacb5ce8da99245343f22fe58e594229b /liblangdet/langdet.h
parent53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff)
downloadlangdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz
langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip
Expose complete language prediction API and simplify CLI
Diffstat (limited to 'liblangdet/langdet.h')
-rw-r--r--liblangdet/langdet.h71
1 files changed, 36 insertions, 35 deletions
diff --git a/liblangdet/langdet.h b/liblangdet/langdet.h
index 9c24264..22e0b64 100644
--- a/liblangdet/langdet.h
+++ b/liblangdet/langdet.h
@@ -3,47 +3,48 @@
#include <stddef.h>
-#define LANGDET_MAX_LAYERS 64
+#ifdef __cplusplus
+extern "C" {
+#endif
-typedef struct {
- int hidden_size;
- int n_layers;
- int n_heads;
- int head_dim;
- int intermediate_size;
- int vocab_size;
- int max_position_embeddings;
- int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
- int type_vocab_size;
- float layer_norm_eps;
-} langdet_config_t;
+#define LANGDET_NUM_LABELS 20
+#define LANGDET_MAX_TOKENS 512
+
+typedef struct langdet langdet_t;
-typedef struct langdet_model langdet_model_t;
-typedef struct langdet_workspace langdet_workspace_t;
-typedef struct langdet_tok langdet_tok_t;
+typedef struct {
+ const char *label;
+ float score;
+ int tokens;
+ int truncated;
+ /* Label pointers belong to the detector and remain valid until langdet_free().
+ * Scores are stored in this result, in the checkpoint's id2label order. */
+ const char *labels[LANGDET_NUM_LABELS];
+ float scores[LANGDET_NUM_LABELS];
+} langdet_result_t;
-/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
-langdet_model_t *langdet_model_load(const char *model_dir);
-void langdet_model_free(langdet_model_t *model);
-const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+/* Library calls must not overlap across application threads, even for different
+ * handles. Internal CPU workers cooperate on one prediction at a time. */
-/* A workspace belongs to one model and may serve only one inference call at a time. */
-langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
-void langdet_workspace_free(langdet_workspace_t *ws);
+/* Load the model, tokenizer, classifier, and labels from model_dir.
+ * n_threads is 1..16, or 0 to detect the CPU count and cap it at 16.
+ * Return NULL on failure. An optional err[errlen] receives a NUL-terminated
+ * message on failure and is cleared on success. Model-file readers may also
+ * write diagnostics to stderr. */
+langdet_t *langdet_open(const char *model_dir, int n_threads, char *err, size_t errlen);
-/* Write the first token's final hidden state to out_embedding[hidden_size].
- * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
-int langdet_model_encode_into(const langdet_model_t *model,
- langdet_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
+/* Classify NUL-terminated UTF-8 text. Truncate to 512 tokens, retaining </s>.
+ * Empty text is accepted but does not yield a meaningful language prediction.
+ * Return 0 on success, or -1 with a zeroed result on failure. The optional
+ * error buffer follows the same contract as langdet_open(). */
+int langdet_predict(langdet_t *detector, const char *text, langdet_result_t *result,
+ char *err, size_t errlen);
-langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
-void langdet_tok_free(langdet_tok_t *t);
+/* Release the detector and its resources. NULL is accepted. */
+void langdet_free(langdet_t *detector);
-/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
- * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
-int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+#ifdef __cplusplus
+}
+#endif
#endif /* LANGDET_H */