aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:11:00 +0200
commit37cbaef9ceb828c667da8f398027ba5415ee7475 (patch)
tree6103e92a9030766e78a35df00f7a1155d0d0ecf5
parentcf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff)
downloadlangdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz
langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip
Rename CPU library API and sources to langdet
-rw-r--r--.gitignore1
-rw-r--r--Makefile22
-rw-r--r--README.md25
-rw-r--r--langdetect.c24
-rw-r--r--libffwd/config.h8
-rw-r--r--liblangdet/LICENSE (renamed from libffwd/LICENSE)0
-rw-r--r--liblangdet/act.c (renamed from libffwd/act.c)0
-rw-r--r--liblangdet/alloc.c (renamed from libffwd/alloc.c)12
-rw-r--r--liblangdet/alloc.h (renamed from libffwd/alloc.h)6
-rw-r--r--liblangdet/attn.c (renamed from libffwd/attn.c)0
-rw-r--r--liblangdet/config.c (renamed from libffwd/config.c)12
-rw-r--r--liblangdet/config.h8
-rw-r--r--liblangdet/forward.c (renamed from libffwd/forward.c)16
-rw-r--r--liblangdet/gemm.c (renamed from libffwd/gemm.c)0
-rw-r--r--liblangdet/generic.c (renamed from libffwd/generic.c)0
-rw-r--r--liblangdet/impl.h (renamed from libffwd/impl.h)0
-rw-r--r--liblangdet/kernels.h (renamed from libffwd/kernels.h)0
-rw-r--r--liblangdet/langdet.h (renamed from libffwd/ffwd.h)36
-rw-r--r--liblangdet/model.c (renamed from libffwd/model.c)44
-rw-r--r--liblangdet/model_blas.c (renamed from libffwd/model_blas.c)10
-rw-r--r--liblangdet/model_internal.h (renamed from libffwd/model_internal.h)10
-rw-r--r--liblangdet/model_types.h (renamed from libffwd/model_types.h)28
-rw-r--r--liblangdet/norm.c (renamed from libffwd/norm.c)0
-rw-r--r--liblangdet/safetensors.c (renamed from libffwd/safetensors.c)0
-rw-r--r--liblangdet/safetensors.h (renamed from libffwd/safetensors.h)0
-rw-r--r--liblangdet/sentencepiece.c (renamed from libffwd/sentencepiece.c)0
-rw-r--r--liblangdet/sentencepiece.h (renamed from libffwd/sentencepiece.h)0
-rw-r--r--liblangdet/threadpool.c (renamed from libffwd/threadpool.c)0
-rw-r--r--liblangdet/threadpool.h (renamed from libffwd/threadpool.h)0
-rw-r--r--liblangdet/tokenize.c (renamed from libffwd/tokenize.c)14
30 files changed, 150 insertions, 126 deletions
diff --git a/.gitignore b/.gitignore
index 9d40c3e..e23dc2a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
/__pycache__/
/build/
/langdetect
+/liblangdet.a
diff --git a/Makefile b/Makefile
index 56e17a0..4ac4306 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
CC = cc
-CPPFLAGS += -Ilibffwd -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK
+AR ?= ar
+CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK
CFLAGS ?= -O3 -march=native
CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread
LDLIBS += -framework Accelerate -lm -pthread
@@ -7,20 +8,25 @@ LDLIBS += -framework Accelerate -lm -pthread
CPU_SOURCES = model.c model_blas.c forward.c alloc.c config.c \
tokenize.c sentencepiece.c safetensors.c \
act.c gemm.c norm.c attn.c threadpool.c generic.c
-SOURCES = langdetect.c $(addprefix libffwd/,$(CPU_SOURCES)) deps/yyjson/yyjson.c
-OBJECTS = $(SOURCES:%.c=build/%.o)
+LIB_SOURCES = $(addprefix liblangdet/,$(CPU_SOURCES)) deps/yyjson/yyjson.c
+LIB_OBJECTS = $(LIB_SOURCES:%.c=build/%.o)
+OBJECTS = build/langdetect.o $(LIB_OBJECTS)
.PHONY: all clean
-all: langdetect
+all: langdetect liblangdet.a
-langdetect: $(OBJECTS)
- $(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LDLIBS)
+langdetect: build/langdetect.o liblangdet.a
+ $(CC) $(LDFLAGS) -o $@ build/langdetect.o liblangdet.a $(LDLIBS)
-build/%.o: %.c
+liblangdet.a: $(LIB_OBJECTS) Makefile
+ rm -f $@
+ $(AR) rcs $@ $(LIB_OBJECTS)
+
+build/%.o: %.c Makefile
@mkdir -p $(@D)
$(CC) $(CPPFLAGS) $(CFLAGS) -MMD -MP -c $< -o $@
clean:
- rm -rf build langdetect
+ rm -rf build langdetect liblangdet.a
-include $(OBJECTS:.o=.d)
diff --git a/README.md b/README.md
index a7f5aae..e74eab9 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
# XLM-R language detection on macOS CPU
-A native C CLI for `papluca/xlm-roberta-base-language-detection`, using ffwd's
-CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language
+A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the
+liblangdet CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language
predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python
runtime is required for inference.
@@ -31,7 +31,7 @@ when there are no text arguments, produces one JSON line in input order:
- `ms` measures tokenization and inference, excluding model loading and output.
`--all` includes a `scores` object with probabilities for every language.
-`-t N` selects 1 to 16 ffwd kernel threads; the default is the detected CPU count,
+`-t N` selects 1 to 16 CPU kernel threads; the default is the detected CPU count,
capped at 16. Accelerate manages its own BLAS threads. `-h` prints help.
Use `--` before text that begins with a hyphen. Diagnostics go to stderr.
@@ -44,7 +44,7 @@ The encoder uses first-token pooling without L2 normalization.
The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer,
and stable softmax. Dropout is disabled during inference.
-The CPU encoder and tokenizer are in `libffwd/`; the JSON parser is in
+The CPU encoder and tokenizer are in `liblangdet/`; the JSON parser is in
`deps/yyjson/`. Their license files are included. The build uses only local source
files and the macOS SDK, with no dependency on another repository.
@@ -54,6 +54,23 @@ accepted, but their predictions are not useful. The model always chooses among
its 20 labels; it has no unknown-language result, and scores are not a guarantee
of accuracy on ambiguous or unsupported text.
+## CPU library
+
+`make` builds both `langdetect` and the static archive `liblangdet.a`. The public
+encoder and tokenizer API is declared in `liblangdet/langdet.h`, with the
+`langdet_` prefix. Each encoder call returns the first token's final hidden state;
+the language-classification head and JSON output remain in `langdetect.c`.
+
+Link a C application with:
+
+```sh
+cc -Iliblangdet application.c liblangdet.a -framework Accelerate -lm -pthread -o application
+```
+
+The archive includes its JSON parser. No separate parser library is needed.
+Models own their memory-mapped weights; free their workspaces before freeing the
+model. A tokenizer or workspace must not be used concurrently by multiple calls.
+
## Download
Python 3 and internet access are the only requirements. No Hugging Face token or
diff --git a/langdetect.c b/langdetect.c
index 003692d..6418d2d 100644
--- a/langdetect.c
+++ b/langdetect.c
@@ -17,9 +17,9 @@
#define MAX_TOKENS 512
typedef struct {
- ffwd_model_t *model;
- ffwd_workspace_t *workspace;
- ffwd_tok_t *tokenizer;
+ langdet_model_t *model;
+ langdet_workspace_t *workspace;
+ langdet_tok_t *tokenizer;
yyjson_doc *config;
const char *labels[NUM_LABELS];
const float *dense_weight, *dense_bias;
@@ -46,9 +46,9 @@ static const float *load_tensor(detector_t *d, const char *name, const int64_t *
}
static void detector_free(detector_t *d) {
- ffwd_tok_free(d->tokenizer);
- ffwd_workspace_free(d->workspace);
- ffwd_model_free(d->model);
+ langdet_tok_free(d->tokenizer);
+ langdet_workspace_free(d->workspace);
+ langdet_model_free(d->model);
yyjson_doc_free(d->config);
}
@@ -87,10 +87,10 @@ static int detector_load(detector_t *d, const char *directory) {
}
}
- d->model = ffwd_model_load(directory);
+ d->model = langdet_model_load(directory);
if (!d->model)
return -1;
- const ffwd_config_t *cfg = ffwd_model_config(d->model);
+ const langdet_config_t *cfg = langdet_model_config(d->model);
if (cfg->hidden_size != HIDDEN_SIZE ||
cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) {
fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n");
@@ -106,12 +106,12 @@ static int detector_load(detector_t *d, const char *directory) {
return -1;
char message[256] = {0};
- d->tokenizer = ffwd_tok_open(directory, message, sizeof(message));
+ d->tokenizer = langdet_tok_open(directory, message, sizeof(message));
if (!d->tokenizer) {
fprintf(stderr, "Tokenizer: %s\n", message);
return -1;
}
- d->workspace = ffwd_workspace_new(d->model);
+ d->workspace = langdet_workspace_new(d->model);
if (!d->workspace) {
fprintf(stderr, "Cannot allocate inference workspace\n");
return -1;
@@ -122,7 +122,7 @@ static int detector_load(detector_t *d, const char *directory) {
static int classify(detector_t *d, const char *text, int all_scores) {
double start = now_ms();
int count = 0;
- int *ids = ffwd_tokenize(d->tokenizer, text, &count);
+ int *ids = langdet_tokenize(d->tokenizer, text, &count);
if (!ids || count < 2) {
fprintf(stderr, "Tokenization failed\n");
free(ids);
@@ -134,7 +134,7 @@ static int classify(detector_t *d, const char *text, int all_scores) {
count = MAX_TOKENS;
}
float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS];
- int result = ffwd_model_encode_into(d->model, d->workspace, ids, count, hidden);
+ int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden);
free(ids);
if (result != 0) {
fprintf(stderr, "Encoder inference failed\n");
diff --git a/libffwd/config.h b/libffwd/config.h
deleted file mode 100644
index b3e2f67..0000000
--- a/libffwd/config.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef FFWD_CONFIG_H
-#define FFWD_CONFIG_H
-
-#include "ffwd.h"
-
-int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir);
-
-#endif /* FFWD_CONFIG_H */
diff --git a/libffwd/LICENSE b/liblangdet/LICENSE
index a00caa7..a00caa7 100644
--- a/libffwd/LICENSE
+++ b/liblangdet/LICENSE
diff --git a/libffwd/act.c b/liblangdet/act.c
index 3912f2f..3912f2f 100644
--- a/libffwd/act.c
+++ b/liblangdet/act.c
diff --git a/libffwd/alloc.c b/liblangdet/alloc.c
index ba21fbb..4bc1496 100644
--- a/libffwd/alloc.c
+++ b/liblangdet/alloc.c
@@ -9,8 +9,8 @@
* - never allocate less than the minimum required size.
* - round up to the granularity so small length changes do not trigger a realloc on every call.
* - keep existing capacity if it is already larger. */
-#define FFWD_MIN_WORKSPACE_SEQ_CAP 16
-#define FFWD_WORKSPACE_SEQ_GRANULARITY 16
+#define LANGDET_MIN_WORKSPACE_SEQ_CAP 16
+#define LANGDET_WORKSPACE_SEQ_GRANULARITY 16
int mul_size(size_t a, size_t b, size_t *out) {
if (a != 0 && b > SIZE_MAX / a)
@@ -24,12 +24,12 @@ int grow_cap(int current, int needed, int *out) {
return -1;
int cap = needed;
- if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP)
- cap = FFWD_MIN_WORKSPACE_SEQ_CAP;
+ if (cap < LANGDET_MIN_WORKSPACE_SEQ_CAP)
+ cap = LANGDET_MIN_WORKSPACE_SEQ_CAP;
- int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY;
+ int rem = cap % LANGDET_WORKSPACE_SEQ_GRANULARITY;
if (rem != 0) {
- int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem;
+ int pad = LANGDET_WORKSPACE_SEQ_GRANULARITY - rem;
if (cap > INT_MAX - pad)
return -1;
cap += pad;
diff --git a/libffwd/alloc.h b/liblangdet/alloc.h
index d3ef81a..f70f1f7 100644
--- a/libffwd/alloc.h
+++ b/liblangdet/alloc.h
@@ -1,5 +1,5 @@
-#ifndef FFWD_ALLOC_H
-#define FFWD_ALLOC_H
+#ifndef LANGDET_ALLOC_H
+#define LANGDET_ALLOC_H
#include <stddef.h>
@@ -19,4 +19,4 @@ int realloc_floats(float **ptr, size_t count);
/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */
int realloc_floats_2d(float **ptr, int rows, int cols);
-#endif /* FFWD_ALLOC_H */
+#endif /* LANGDET_ALLOC_H */
diff --git a/libffwd/attn.c b/liblangdet/attn.c
index ac0f4d1..ac0f4d1 100644
--- a/libffwd/attn.c
+++ b/liblangdet/attn.c
diff --git a/libffwd/config.c b/liblangdet/config.c
index 6343113..f22a9d7 100644
--- a/libffwd/config.c
+++ b/liblangdet/config.c
@@ -16,18 +16,18 @@ static int json_get_int(yyjson_val *obj, const char *key, int fallback) {
return n >= 0 && n <= INT_MAX ? (int)n : -1;
}
-int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) {
+int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "ffwd_config: model path too long\n");
+ fprintf(stderr, "langdet_config: model path too long\n");
return -1;
}
yyjson_read_err error;
yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error);
if (!doc) {
- fprintf(stderr, "ffwd_config: cannot read %s: %s\n", path, error.msg);
+ fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg);
return -1;
}
yyjson_val *root = yyjson_doc_get_root(doc);
@@ -35,7 +35,7 @@ int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) {
if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") ||
!yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") ||
(causal && !yyjson_is_false(causal))) {
- fprintf(stderr, "ffwd_config: expected bidirectional XLM-R with GeLU activation\n");
+ fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n");
yyjson_doc_free(doc);
return -1;
}
@@ -54,13 +54,13 @@ int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) {
cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1;
yyjson_doc_free(doc);
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > FFWD_MAX_LAYERS ||
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS ||
cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 ||
cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 ||
cfg->position_id_offset < 0 ||
cfg->max_position_embeddings <= cfg->position_id_offset ||
!isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) {
- fprintf(stderr, "ffwd_config: invalid encoder dimensions or normalization in %s\n", path);
+ fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path);
return -1;
}
return 0;
diff --git a/liblangdet/config.h b/liblangdet/config.h
new file mode 100644
index 0000000..79522fc
--- /dev/null
+++ b/liblangdet/config.h
@@ -0,0 +1,8 @@
+#ifndef LANGDET_CONFIG_H
+#define LANGDET_CONFIG_H
+
+#include "langdet.h"
+
+int langdet_config_parse(langdet_config_t *cfg, const char *model_dir);
+
+#endif /* LANGDET_CONFIG_H */
diff --git a/libffwd/forward.c b/liblangdet/forward.c
index b314642..14b1327 100644
--- a/libffwd/forward.c
+++ b/liblangdet/forward.c
@@ -14,16 +14,16 @@ static void add_bias_rows(float *x, const float *bias, int rows, int dim) {
}
}
-int ffwd_model_encode_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
const int *token_ids,
int n_tokens,
float *out_embedding) {
if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding)
return -1;
- const ffwd_config_t *cfg = &model->config;
- const ffwd_weights_t *w = &model->weights;
+ const langdet_config_t *cfg = &model->config;
+ const langdet_weights_t *w = &model->weights;
int hidden = cfg->hidden_size;
int n_heads = cfg->n_heads;
int head_dim = cfg->head_dim;
@@ -32,7 +32,7 @@ int ffwd_model_encode_into(const ffwd_model_t *model,
if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) {
fprintf(stderr,
- "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings "
+ "langdet: sequence length %d plus position offset %d exceeds max_position_embeddings "
"%d\n",
n_tokens, cfg->position_id_offset, cfg->max_position_embeddings);
return -1;
@@ -50,7 +50,7 @@ int ffwd_model_encode_into(const ffwd_model_t *model,
for (int i = 0; i < n_tokens; i++) {
int id = token_ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "ffwd: invalid token id %d at position %d\n", id, i);
+ fprintf(stderr, "langdet: invalid token id %d at position %d\n", id, i);
return -1;
}
float *row = x + (size_t)i * hidden;
@@ -59,12 +59,12 @@ int ffwd_model_encode_into(const ffwd_model_t *model,
add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
add_inplace(row, w->token_type_embeddings, hidden);
}
- layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, n_tokens, hidden, eps);
+ layer_norm(x, x, w->langdet_ln_w, w->langdet_ln_b, n_tokens, hidden, eps);
float scale = 1.0f / sqrtf((float)head_dim);
for (int layer = 0; layer < cfg->n_layers; layer++) {
- const ffwd_layer_t *l = &w->layers[layer];
+ const langdet_layer_t *l = &w->layers[layer];
linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden);
linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden);
diff --git a/libffwd/gemm.c b/liblangdet/gemm.c
index c750be1..c750be1 100644
--- a/libffwd/gemm.c
+++ b/liblangdet/gemm.c
diff --git a/libffwd/generic.c b/liblangdet/generic.c
index 729c610..729c610 100644
--- a/libffwd/generic.c
+++ b/liblangdet/generic.c
diff --git a/libffwd/impl.h b/liblangdet/impl.h
index f216518..f216518 100644
--- a/libffwd/impl.h
+++ b/liblangdet/impl.h
diff --git a/libffwd/kernels.h b/liblangdet/kernels.h
index 1025602..1025602 100644
--- a/libffwd/kernels.h
+++ b/liblangdet/kernels.h
diff --git a/libffwd/ffwd.h b/liblangdet/langdet.h
index 527e92f..9c24264 100644
--- a/libffwd/ffwd.h
+++ b/liblangdet/langdet.h
@@ -1,9 +1,9 @@
-#ifndef FFWD_H
-#define FFWD_H
+#ifndef LANGDET_H
+#define LANGDET_H
#include <stddef.h>
-#define FFWD_MAX_LAYERS 64
+#define LANGDET_MAX_LAYERS 64
typedef struct {
int hidden_size;
@@ -16,34 +16,34 @@ typedef struct {
int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */
int type_vocab_size;
float layer_norm_eps;
-} ffwd_config_t;
+} langdet_config_t;
-typedef struct ffwd_model ffwd_model_t;
-typedef struct ffwd_workspace ffwd_workspace_t;
-typedef struct ffwd_tok ffwd_tok_t;
+typedef struct langdet_model langdet_model_t;
+typedef struct langdet_workspace langdet_workspace_t;
+typedef struct langdet_tok langdet_tok_t;
/* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */
-ffwd_model_t *ffwd_model_load(const char *model_dir);
-void ffwd_model_free(ffwd_model_t *model);
-const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model);
+langdet_model_t *langdet_model_load(const char *model_dir);
+void langdet_model_free(langdet_model_t *model);
+const langdet_config_t *langdet_model_config(const langdet_model_t *model);
/* A workspace belongs to one model and may serve only one inference call at a time. */
-ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model);
-void ffwd_workspace_free(ffwd_workspace_t *ws);
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model);
+void langdet_workspace_free(langdet_workspace_t *ws);
/* Write the first token's final hidden state to out_embedding[hidden_size].
* Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */
-int ffwd_model_encode_into(const ffwd_model_t *model,
- ffwd_workspace_t *ws,
+int langdet_model_encode_into(const langdet_model_t *model,
+ langdet_workspace_t *ws,
const int *token_ids,
int n_tokens,
float *out_embedding);
-ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen);
-void ffwd_tok_free(ffwd_tok_t *t);
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen);
+void langdet_tok_free(langdet_tok_t *t);
/* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error.
* The tokenizer owns reusable scratch and is not safe for concurrent calls. */
-int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out);
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out);
-#endif /* FFWD_H */
+#endif /* LANGDET_H */
diff --git a/libffwd/model.c b/liblangdet/model.c
index 81073f3..146bbe7 100644
--- a/libffwd/model.c
+++ b/liblangdet/model.c
@@ -10,11 +10,11 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
const int64_t *shape,
int ndim) {
if (t->dtype != DTYPE_F32) {
- fprintf(stderr, "ffwd: expected F32 tensor: %s\n", name);
+ fprintf(stderr, "langdet: expected F32 tensor: %s\n", name);
return 0;
}
if (t->ndim != ndim) {
- fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
+ fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
return 0;
}
@@ -22,25 +22,25 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
for (int i = 0; i < ndim; i++) {
if (t->shape[i] != shape[i]) {
fprintf(stderr,
- "ffwd: bad shape for %s at dim %d: got %lld, expected %lld\n",
+ "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n",
name, i, (long long)t->shape[i], (long long)shape[i]);
return 0;
}
if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) {
- fprintf(stderr, "ffwd: shape too large or empty for %s\n", name);
+ fprintf(stderr, "langdet: shape too large or empty for %s\n", name);
return 0;
}
numel *= (size_t)shape[i];
}
size_t bytes;
if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) {
- fprintf(stderr, "ffwd: invalid data size for %s\n", name);
+ fprintf(stderr, "langdet: invalid data size for %s\n", name);
return 0;
}
size_t data_start = 8 + sf->header_size;
if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
t->data_size > sf->file_size - data_start - t->data_offset) {
- fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name);
+ fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name);
return 0;
}
return 1;
@@ -51,7 +51,7 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name,
safetensors_file_t *sf = NULL;
const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
- fprintf(stderr, "ffwd: tensor not found: %s\n", name);
+ fprintf(stderr, "langdet: tensor not found: %s\n", name);
return NULL;
}
if (!tensor_has_supported_shape(sf, t, name, shape, ndim))
@@ -59,9 +59,9 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name,
return safetensors_data(sf, t);
}
-static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) {
- const ffwd_config_t *cfg = &model->config;
- ffwd_weights_t *w = &model->weights;
+static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
+ const langdet_config_t *cfg = &model->config;
+ langdet_weights_t *w = &model->weights;
char name[256];
int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
@@ -71,18 +71,18 @@ static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) {
(int64_t[]){cfg->max_position_embeddings, hidden}, 2);
w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
(int64_t[]){cfg->type_vocab_size, hidden}, 2);
- w->ffwd_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
- w->ffwd_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
+ w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
+ w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
- !w->ffwd_ln_w || !w->ffwd_ln_b)
+ !w->langdet_ln_w || !w->langdet_ln_b)
return -1;
- w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t));
+ w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t));
if (!w->layers)
return -1;
for (int i = 0; i < cfg->n_layers; i++) {
- ffwd_layer_t *l = &w->layers[i];
+ langdet_layer_t *l = &w->layers[i];
#define BERT_W(field, suffix, d0, d1) \
do { \
@@ -124,17 +124,17 @@ static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) {
return 0;
}
-ffwd_model_t *ffwd_model_load(const char *model_dir) {
+langdet_model_t *langdet_model_load(const char *model_dir) {
if (!model_dir)
return NULL;
- ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t));
+ langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t));
if (!model)
return NULL;
- if (ffwd_config_parse(&model->config, model_dir) != 0)
+ if (langdet_config_parse(&model->config, model_dir) != 0)
goto fail;
model->safetensors = multi_safetensors_open(model_dir);
if (!model->safetensors) {
- fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir);
+ fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir);
goto fail;
}
if (load_bert_weights(model, model->safetensors) != 0)
@@ -142,11 +142,11 @@ ffwd_model_t *ffwd_model_load(const char *model_dir) {
return model;
fail:
- ffwd_model_free(model);
+ langdet_model_free(model);
return NULL;
}
-void ffwd_model_free(ffwd_model_t *model) {
+void langdet_model_free(langdet_model_t *model) {
if (!model)
return;
multi_safetensors_close(model->safetensors);
@@ -154,6 +154,6 @@ void ffwd_model_free(ffwd_model_t *model) {
free(model);
}
-const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) {
+const langdet_config_t *langdet_model_config(const langdet_model_t *model) {
return model ? &model->config : NULL;
}
diff --git a/libffwd/model_blas.c b/liblangdet/model_blas.c
index 409ba66..41f9237 100644
--- a/libffwd/model_blas.c
+++ b/liblangdet/model_blas.c
@@ -3,17 +3,17 @@
#include <stdlib.h>
-ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) {
+langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model) {
if (!model)
return NULL;
- ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t));
+ langdet_workspace_t *ws = (langdet_workspace_t *)calloc(1, sizeof(langdet_workspace_t));
if (!ws)
return NULL;
ws->model = model;
return ws;
}
-void ffwd_workspace_free(ffwd_workspace_t *ws) {
+void langdet_workspace_free(langdet_workspace_t *ws) {
if (!ws)
return;
free(ws->x);
@@ -27,7 +27,7 @@ void ffwd_workspace_free(ffwd_workspace_t *ws) {
free(ws);
}
-int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) {
+int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq) {
if (seq <= ws->buf_seq_cap)
return 0;
@@ -55,7 +55,7 @@ int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) {
return 0;
}
-void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) {
+void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) {
size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
if (bytes <= ws->attn_scores_bytes)
return;
diff --git a/libffwd/model_internal.h b/liblangdet/model_internal.h
index aeea52e..73a7851 100644
--- a/libffwd/model_internal.h
+++ b/liblangdet/model_internal.h
@@ -1,5 +1,5 @@
-#ifndef FFWD_MODEL_INTERNAL_H
-#define FFWD_MODEL_INTERNAL_H
+#ifndef LANGDET_MODEL_INTERNAL_H
+#define LANGDET_MODEL_INTERNAL_H
#include "model_types.h"
#include "alloc.h"
@@ -11,7 +11,7 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
const int64_t *shape,
int ndim);
-int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int seq);
-void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch);
+int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq);
+void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch);
-#endif /* FFWD_MODEL_INTERNAL_H */
+#endif /* LANGDET_MODEL_INTERNAL_H */
diff --git a/libffwd/model_types.h b/liblangdet/model_types.h
index 447b653..2170909 100644
--- a/libffwd/model_types.h
+++ b/liblangdet/model_types.h
@@ -1,7 +1,7 @@
-#ifndef FFWD_MODEL_TYPES_H
-#define FFWD_MODEL_TYPES_H
+#ifndef LANGDET_MODEL_TYPES_H
+#define LANGDET_MODEL_TYPES_H
-#include "ffwd.h"
+#include "langdet.h"
#include "safetensors.h"
typedef struct {
@@ -21,26 +21,26 @@ typedef struct {
const float *ffn_out_bias;
const float *post_attn_norm;
const float *ffn_ln_bias;
-} ffwd_layer_t;
+} langdet_layer_t;
typedef struct {
const float *embed_tokens;
- ffwd_layer_t *layers;
+ langdet_layer_t *layers;
const float *position_embeddings;
const float *token_type_embeddings;
- const float *ffwd_ln_w;
- const float *ffwd_ln_b;
-} ffwd_weights_t;
+ const float *langdet_ln_w;
+ const float *langdet_ln_b;
+} langdet_weights_t;
-struct ffwd_model {
- ffwd_config_t config;
- ffwd_weights_t weights;
+struct langdet_model {
+ langdet_config_t config;
+ langdet_weights_t weights;
/* All weight pointers borrow this mapping until model teardown. */
multi_safetensors_t *safetensors;
};
-struct ffwd_workspace {
- const ffwd_model_t *model;
+struct langdet_workspace {
+ const langdet_model_t *model;
int buf_seq_cap;
float *x;
float *q;
@@ -53,4 +53,4 @@ struct ffwd_workspace {
size_t attn_scores_bytes;
};
-#endif /* FFWD_MODEL_TYPES_H */
+#endif /* LANGDET_MODEL_TYPES_H */
diff --git a/libffwd/norm.c b/liblangdet/norm.c
index 59e8bdb..59e8bdb 100644
--- a/libffwd/norm.c
+++ b/liblangdet/norm.c
diff --git a/libffwd/safetensors.c b/liblangdet/safetensors.c
index cfba76e..cfba76e 100644
--- a/libffwd/safetensors.c
+++ b/liblangdet/safetensors.c
diff --git a/libffwd/safetensors.h b/liblangdet/safetensors.h
index dca68a7..dca68a7 100644
--- a/libffwd/safetensors.h
+++ b/liblangdet/safetensors.h
diff --git a/libffwd/sentencepiece.c b/liblangdet/sentencepiece.c
index 6c30ba1..6c30ba1 100644
--- a/libffwd/sentencepiece.c
+++ b/liblangdet/sentencepiece.c
diff --git a/libffwd/sentencepiece.h b/liblangdet/sentencepiece.h
index 2230538..2230538 100644
--- a/libffwd/sentencepiece.h
+++ b/liblangdet/sentencepiece.h
diff --git a/libffwd/threadpool.c b/liblangdet/threadpool.c
index 6cb4252..6cb4252 100644
--- a/libffwd/threadpool.c
+++ b/liblangdet/threadpool.c
diff --git a/libffwd/threadpool.h b/liblangdet/threadpool.h
index f9739a1..f9739a1 100644
--- a/libffwd/threadpool.h
+++ b/liblangdet/threadpool.h
diff --git a/libffwd/tokenize.c b/liblangdet/tokenize.c
index 8e1000f..5e26a18 100644
--- a/libffwd/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -1,4 +1,4 @@
-#include "ffwd.h"
+#include "langdet.h"
#include "sentencepiece.h"
#include <limits.h>
@@ -6,7 +6,7 @@
#include <stdlib.h>
#include <string.h>
-struct ffwd_tok {
+struct langdet_tok {
tok_spm_t *spm;
tok_spm_workspace_t *workspace;
int cls_id;
@@ -18,12 +18,12 @@ static void set_err(char *err, size_t errlen, const char *msg) {
snprintf(err, errlen, "%s", msg);
}
-ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) {
+langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) {
if (!model_dir) {
set_err(err, errlen, "model_dir is NULL");
return NULL;
}
- ffwd_tok_t *t = calloc(1, sizeof(*t));
+ langdet_tok_t *t = calloc(1, sizeof(*t));
if (!t) {
set_err(err, errlen, "out of memory");
return NULL;
@@ -47,11 +47,11 @@ ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) {
return t;
fail:
- ffwd_tok_free(t);
+ langdet_tok_free(t);
return NULL;
}
-void ffwd_tok_free(ffwd_tok_t *t) {
+void langdet_tok_free(langdet_tok_t *t) {
if (!t)
return;
tok_spm_workspace_free(t->workspace);
@@ -59,7 +59,7 @@ void ffwd_tok_free(ffwd_tok_t *t) {
free(t);
}
-int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) {
+int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
if (n_out)
*n_out = 0;
if (!t || !text)