aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/encoder.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:40:12 +0200
commitcb1a04331b27cc19445de63cedcd4fd30b8a9016 (patch)
tree85edb98bacb5ce8da99245343f22fe58e594229b /liblangdet/encoder.h
parent53c1b6591b5d237637cbb7bd251cfdcf9f4a554a (diff)
downloadlangdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.tar.gz
langdet-cb1a04331b27cc19445de63cedcd4fd30b8a9016.zip
Expose complete language prediction API and simplify CLI
Diffstat (limited to 'liblangdet/encoder.h')
-rw-r--r--liblangdet/encoder.h49
1 files changed, 49 insertions, 0 deletions
diff --git a/liblangdet/encoder.h b/liblangdet/encoder.h
new file mode 100644
index 0000000..7d954df
--- /dev/null
+++ b/liblangdet/encoder.h
@@ -0,0 +1,49 @@
+#ifndef LANGDET_ENCODER_H
+#define LANGDET_ENCODER_H
+
+#include <stddef.h>
+
+#define LANGDET_MAX_LAYERS 64
+
+typedef struct {
+ int hidden_size;
+ int n_layers;
+ int n_heads;
+ int head_dim;
+ int intermediate_size;
+ int vocab_size;
+ int max_position_embeddings;
+ int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
+ int type_vocab_size;
+ float layer_norm_eps;
+} langdet_config_t;
+
+typedef struct langdet_model langdet_model_t;
+typedef struct langdet_workspace langdet_workspace_t;
+typedef struct langdet_tok langdet_tok_t;
+
+/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
+langdet_model_t *langdet_model_load(const char *model_dir);
+void langdet_model_free(langdet_model_t *model);
+const langdet_config_t *langdet_model_config(const langdet_model_t *model);
+
+/* A workspace belongs to one model and may serve only one inference call at a time. */
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
+void langdet_workspace_free(langdet_workspace_t *ws);
+
+/* Write the first token's final hidden state to out_embedding[hidden_size].
+ * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding);
+
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
+void langdet_tok_free(langdet_tok_t *t);
+
+/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
+ * The tokenizer owns reusable scratch and is not safe for concurrent calls. */
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
+
+#endif /* LANGDET_ENCODER_H */