diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 13:11:00 +0200 |
| commit | 37cbaef9ceb828c667da8f398027ba5415ee7475 (patch) | |
| tree | 6103e92a9030766e78a35df00f7a1155d0d0ecf5 | |
| parent | cf7d8169ec12cb3adf90d4ff78b3257e74985302 (diff) | |
| download | langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.tar.gz langdet-37cbaef9ceb828c667da8f398027ba5415ee7475.zip | |
Rename CPU library API and sources to langdet
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 22 | ||||
| -rw-r--r-- | README.md | 25 | ||||
| -rw-r--r-- | langdetect.c | 24 | ||||
| -rw-r--r-- | libffwd/config.h | 8 | ||||
| -rw-r--r-- | liblangdet/LICENSE (renamed from libffwd/LICENSE) | 0 | ||||
| -rw-r--r-- | liblangdet/act.c (renamed from libffwd/act.c) | 0 | ||||
| -rw-r--r-- | liblangdet/alloc.c (renamed from libffwd/alloc.c) | 12 | ||||
| -rw-r--r-- | liblangdet/alloc.h (renamed from libffwd/alloc.h) | 6 | ||||
| -rw-r--r-- | liblangdet/attn.c (renamed from libffwd/attn.c) | 0 | ||||
| -rw-r--r-- | liblangdet/config.c (renamed from libffwd/config.c) | 12 | ||||
| -rw-r--r-- | liblangdet/config.h | 8 | ||||
| -rw-r--r-- | liblangdet/forward.c (renamed from libffwd/forward.c) | 16 | ||||
| -rw-r--r-- | liblangdet/gemm.c (renamed from libffwd/gemm.c) | 0 | ||||
| -rw-r--r-- | liblangdet/generic.c (renamed from libffwd/generic.c) | 0 | ||||
| -rw-r--r-- | liblangdet/impl.h (renamed from libffwd/impl.h) | 0 | ||||
| -rw-r--r-- | liblangdet/kernels.h (renamed from libffwd/kernels.h) | 0 | ||||
| -rw-r--r-- | liblangdet/langdet.h (renamed from libffwd/ffwd.h) | 36 | ||||
| -rw-r--r-- | liblangdet/model.c (renamed from libffwd/model.c) | 44 | ||||
| -rw-r--r-- | liblangdet/model_blas.c (renamed from libffwd/model_blas.c) | 10 | ||||
| -rw-r--r-- | liblangdet/model_internal.h (renamed from libffwd/model_internal.h) | 10 | ||||
| -rw-r--r-- | liblangdet/model_types.h (renamed from libffwd/model_types.h) | 28 | ||||
| -rw-r--r-- | liblangdet/norm.c (renamed from libffwd/norm.c) | 0 | ||||
| -rw-r--r-- | liblangdet/safetensors.c (renamed from libffwd/safetensors.c) | 0 | ||||
| -rw-r--r-- | liblangdet/safetensors.h (renamed from libffwd/safetensors.h) | 0 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.c (renamed from libffwd/sentencepiece.c) | 0 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.h (renamed from libffwd/sentencepiece.h) | 0 | ||||
| -rw-r--r-- | liblangdet/threadpool.c (renamed from libffwd/threadpool.c) | 0 | ||||
| -rw-r--r-- | liblangdet/threadpool.h (renamed from libffwd/threadpool.h) | 0 | ||||
| -rw-r--r-- | liblangdet/tokenize.c (renamed from libffwd/tokenize.c) | 14 |
30 files changed, 150 insertions, 126 deletions
@@ -2,3 +2,4 @@ /__pycache__/ /build/ /langdetect +/liblangdet.a @@ -1,5 +1,6 @@ CC = cc -CPPFLAGS += -Ilibffwd -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK +AR ?= ar +CPPFLAGS += -Iliblangdet -Ideps/yyjson -DUSE_BLAS -DACCELERATE_NEW_LAPACK CFLAGS ?= -O3 -march=native CFLAGS += -std=c11 -Wall -Wextra -Werror=implicit-function-declaration -pthread LDLIBS += -framework Accelerate -lm -pthread @@ -7,20 +8,25 @@ LDLIBS += -framework Accelerate -lm -pthread CPU_SOURCES = model.c model_blas.c forward.c alloc.c config.c \ tokenize.c sentencepiece.c safetensors.c \ act.c gemm.c norm.c attn.c threadpool.c generic.c -SOURCES = langdetect.c $(addprefix libffwd/,$(CPU_SOURCES)) deps/yyjson/yyjson.c -OBJECTS = $(SOURCES:%.c=build/%.o) +LIB_SOURCES = $(addprefix liblangdet/,$(CPU_SOURCES)) deps/yyjson/yyjson.c +LIB_OBJECTS = $(LIB_SOURCES:%.c=build/%.o) +OBJECTS = build/langdetect.o $(LIB_OBJECTS) .PHONY: all clean -all: langdetect +all: langdetect liblangdet.a -langdetect: $(OBJECTS) - $(CC) $(LDFLAGS) -o $@ $(OBJECTS) $(LDLIBS) +langdetect: build/langdetect.o liblangdet.a + $(CC) $(LDFLAGS) -o $@ build/langdetect.o liblangdet.a $(LDLIBS) -build/%.o: %.c +liblangdet.a: $(LIB_OBJECTS) Makefile + rm -f $@ + $(AR) rcs $@ $(LIB_OBJECTS) + +build/%.o: %.c Makefile @mkdir -p $(@D) $(CC) $(CPPFLAGS) $(CFLAGS) -MMD -MP -c $< -o $@ clean: - rm -rf build langdetect + rm -rf build langdetect liblangdet.a -include $(OBJECTS:.o=.d) @@ -1,7 +1,7 @@ # XLM-R language detection on macOS CPU -A native C CLI for `papluca/xlm-roberta-base-language-detection`, using ffwd's -CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language +A native C CLI for `papluca/xlm-roberta-base-language-detection`, using the +liblangdet CPU encoder, ARM NEON kernels, and Apple Accelerate BLAS. It returns language predictions for the checkpoint's 20 languages. No MLX, Metal, PyTorch, or Python runtime is required for inference. @@ -31,7 +31,7 @@ when there are no text arguments, produces one JSON line in input order: - `ms` measures tokenization and inference, excluding model loading and output. `--all` includes a `scores` object with probabilities for every language. -`-t N` selects 1 to 16 ffwd kernel threads; the default is the detected CPU count, +`-t N` selects 1 to 16 CPU kernel threads; the default is the detected CPU count, capped at 16. Accelerate manages its own BLAS threads. `-h` prints help. Use `--` before text that begins with a hyphen. Diagnostics go to stderr. @@ -44,7 +44,7 @@ The encoder uses first-token pooling without L2 normalization. The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer, and stable softmax. Dropout is disabled during inference. -The CPU encoder and tokenizer are in `libffwd/`; the JSON parser is in +The CPU encoder and tokenizer are in `liblangdet/`; the JSON parser is in `deps/yyjson/`. Their license files are included. The build uses only local source files and the macOS SDK, with no dependency on another repository. @@ -54,6 +54,23 @@ accepted, but their predictions are not useful. The model always chooses among its 20 labels; it has no unknown-language result, and scores are not a guarantee of accuracy on ambiguous or unsupported text. +## CPU library + +`make` builds both `langdetect` and the static archive `liblangdet.a`. The public +encoder and tokenizer API is declared in `liblangdet/langdet.h`, with the +`langdet_` prefix. Each encoder call returns the first token's final hidden state; +the language-classification head and JSON output remain in `langdetect.c`. + +Link a C application with: + +```sh +cc -Iliblangdet application.c liblangdet.a -framework Accelerate -lm -pthread -o application +``` + +The archive includes its JSON parser. No separate parser library is needed. +Models own their memory-mapped weights; free their workspaces before freeing the +model. A tokenizer or workspace must not be used concurrently by multiple calls. + ## Download Python 3 and internet access are the only requirements. No Hugging Face token or diff --git a/langdetect.c b/langdetect.c index 003692d..6418d2d 100644 --- a/langdetect.c +++ b/langdetect.c @@ -17,9 +17,9 @@ #define MAX_TOKENS 512 typedef struct { - ffwd_model_t *model; - ffwd_workspace_t *workspace; - ffwd_tok_t *tokenizer; + langdet_model_t *model; + langdet_workspace_t *workspace; + langdet_tok_t *tokenizer; yyjson_doc *config; const char *labels[NUM_LABELS]; const float *dense_weight, *dense_bias; @@ -46,9 +46,9 @@ static const float *load_tensor(detector_t *d, const char *name, const int64_t * } static void detector_free(detector_t *d) { - ffwd_tok_free(d->tokenizer); - ffwd_workspace_free(d->workspace); - ffwd_model_free(d->model); + langdet_tok_free(d->tokenizer); + langdet_workspace_free(d->workspace); + langdet_model_free(d->model); yyjson_doc_free(d->config); } @@ -87,10 +87,10 @@ static int detector_load(detector_t *d, const char *directory) { } } - d->model = ffwd_model_load(directory); + d->model = langdet_model_load(directory); if (!d->model) return -1; - const ffwd_config_t *cfg = ffwd_model_config(d->model); + const langdet_config_t *cfg = langdet_model_config(d->model); if (cfg->hidden_size != HIDDEN_SIZE || cfg->max_position_embeddings - cfg->position_id_offset < MAX_TOKENS) { fprintf(stderr, "Expected a 768-dimensional XLM-R encoder with 512-token capacity\n"); @@ -106,12 +106,12 @@ static int detector_load(detector_t *d, const char *directory) { return -1; char message[256] = {0}; - d->tokenizer = ffwd_tok_open(directory, message, sizeof(message)); + d->tokenizer = langdet_tok_open(directory, message, sizeof(message)); if (!d->tokenizer) { fprintf(stderr, "Tokenizer: %s\n", message); return -1; } - d->workspace = ffwd_workspace_new(d->model); + d->workspace = langdet_workspace_new(d->model); if (!d->workspace) { fprintf(stderr, "Cannot allocate inference workspace\n"); return -1; @@ -122,7 +122,7 @@ static int detector_load(detector_t *d, const char *directory) { static int classify(detector_t *d, const char *text, int all_scores) { double start = now_ms(); int count = 0; - int *ids = ffwd_tokenize(d->tokenizer, text, &count); + int *ids = langdet_tokenize(d->tokenizer, text, &count); if (!ids || count < 2) { fprintf(stderr, "Tokenization failed\n"); free(ids); @@ -134,7 +134,7 @@ static int classify(detector_t *d, const char *text, int all_scores) { count = MAX_TOKENS; } float hidden[HIDDEN_SIZE], dense[HIDDEN_SIZE], scores[NUM_LABELS]; - int result = ffwd_model_encode_into(d->model, d->workspace, ids, count, hidden); + int result = langdet_model_encode_into(d->model, d->workspace, ids, count, hidden); free(ids); if (result != 0) { fprintf(stderr, "Encoder inference failed\n"); diff --git a/libffwd/config.h b/libffwd/config.h deleted file mode 100644 index b3e2f67..0000000 --- a/libffwd/config.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef FFWD_CONFIG_H -#define FFWD_CONFIG_H - -#include "ffwd.h" - -int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir); - -#endif /* FFWD_CONFIG_H */ diff --git a/libffwd/LICENSE b/liblangdet/LICENSE index a00caa7..a00caa7 100644 --- a/libffwd/LICENSE +++ b/liblangdet/LICENSE diff --git a/libffwd/act.c b/liblangdet/act.c index 3912f2f..3912f2f 100644 --- a/libffwd/act.c +++ b/liblangdet/act.c diff --git a/libffwd/alloc.c b/liblangdet/alloc.c index ba21fbb..4bc1496 100644 --- a/libffwd/alloc.c +++ b/liblangdet/alloc.c @@ -9,8 +9,8 @@ * - never allocate less than the minimum required size. * - round up to the granularity so small length changes do not trigger a realloc on every call. * - keep existing capacity if it is already larger. */ -#define FFWD_MIN_WORKSPACE_SEQ_CAP 16 -#define FFWD_WORKSPACE_SEQ_GRANULARITY 16 +#define LANGDET_MIN_WORKSPACE_SEQ_CAP 16 +#define LANGDET_WORKSPACE_SEQ_GRANULARITY 16 int mul_size(size_t a, size_t b, size_t *out) { if (a != 0 && b > SIZE_MAX / a) @@ -24,12 +24,12 @@ int grow_cap(int current, int needed, int *out) { return -1; int cap = needed; - if (cap < FFWD_MIN_WORKSPACE_SEQ_CAP) - cap = FFWD_MIN_WORKSPACE_SEQ_CAP; + if (cap < LANGDET_MIN_WORKSPACE_SEQ_CAP) + cap = LANGDET_MIN_WORKSPACE_SEQ_CAP; - int rem = cap % FFWD_WORKSPACE_SEQ_GRANULARITY; + int rem = cap % LANGDET_WORKSPACE_SEQ_GRANULARITY; if (rem != 0) { - int pad = FFWD_WORKSPACE_SEQ_GRANULARITY - rem; + int pad = LANGDET_WORKSPACE_SEQ_GRANULARITY - rem; if (cap > INT_MAX - pad) return -1; cap += pad; diff --git a/libffwd/alloc.h b/liblangdet/alloc.h index d3ef81a..f70f1f7 100644 --- a/libffwd/alloc.h +++ b/liblangdet/alloc.h @@ -1,5 +1,5 @@ -#ifndef FFWD_ALLOC_H -#define FFWD_ALLOC_H +#ifndef LANGDET_ALLOC_H +#define LANGDET_ALLOC_H #include <stddef.h> @@ -19,4 +19,4 @@ int realloc_floats(float **ptr, size_t count); /* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */ int realloc_floats_2d(float **ptr, int rows, int cols); -#endif /* FFWD_ALLOC_H */ +#endif /* LANGDET_ALLOC_H */ diff --git a/libffwd/attn.c b/liblangdet/attn.c index ac0f4d1..ac0f4d1 100644 --- a/libffwd/attn.c +++ b/liblangdet/attn.c diff --git a/libffwd/config.c b/liblangdet/config.c index 6343113..f22a9d7 100644 --- a/libffwd/config.c +++ b/liblangdet/config.c @@ -16,18 +16,18 @@ static int json_get_int(yyjson_val *obj, const char *key, int fallback) { return n >= 0 && n <= INT_MAX ? (int)n : -1; } -int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) { +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir) { char path[1024]; int n = snprintf(path, sizeof(path), "%s/config.json", model_dir); if (n < 0 || (size_t)n >= sizeof(path)) { - fprintf(stderr, "ffwd_config: model path too long\n"); + fprintf(stderr, "langdet_config: model path too long\n"); return -1; } yyjson_read_err error; yyjson_doc *doc = yyjson_read_file(path, 0, NULL, &error); if (!doc) { - fprintf(stderr, "ffwd_config: cannot read %s: %s\n", path, error.msg); + fprintf(stderr, "langdet_config: cannot read %s: %s\n", path, error.msg); return -1; } yyjson_val *root = yyjson_doc_get_root(doc); @@ -35,7 +35,7 @@ int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) { if (!yyjson_equals_str(yyjson_obj_get(root, "model_type"), "xlm-roberta") || !yyjson_equals_str(yyjson_obj_get(root, "hidden_act"), "gelu") || (causal && !yyjson_is_false(causal))) { - fprintf(stderr, "ffwd_config: expected bidirectional XLM-R with GeLU activation\n"); + fprintf(stderr, "langdet_config: expected bidirectional XLM-R with GeLU activation\n"); yyjson_doc_free(doc); return -1; } @@ -54,13 +54,13 @@ int ffwd_config_parse(ffwd_config_t *cfg, const char *model_dir) { cfg->position_id_offset = pad_id >= 0 && pad_id < INT_MAX ? pad_id + 1 : -1; yyjson_doc_free(doc); - if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > FFWD_MAX_LAYERS || + if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_layers > LANGDET_MAX_LAYERS || cfg->n_heads <= 0 || cfg->head_dim <= 0 || cfg->hidden_size % cfg->n_heads != 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 || cfg->type_vocab_size <= 0 || cfg->position_id_offset < 0 || cfg->max_position_embeddings <= cfg->position_id_offset || !isfinite(cfg->layer_norm_eps) || cfg->layer_norm_eps <= 0.0f) { - fprintf(stderr, "ffwd_config: invalid encoder dimensions or normalization in %s\n", path); + fprintf(stderr, "langdet_config: invalid encoder dimensions or normalization in %s\n", path); return -1; } return 0; diff --git a/liblangdet/config.h b/liblangdet/config.h new file mode 100644 index 0000000..79522fc --- /dev/null +++ b/liblangdet/config.h @@ -0,0 +1,8 @@ +#ifndef LANGDET_CONFIG_H +#define LANGDET_CONFIG_H + +#include "langdet.h" + +int langdet_config_parse(langdet_config_t *cfg, const char *model_dir); + +#endif /* LANGDET_CONFIG_H */ diff --git a/libffwd/forward.c b/liblangdet/forward.c index b314642..14b1327 100644 --- a/libffwd/forward.c +++ b/liblangdet/forward.c @@ -14,16 +14,16 @@ static void add_bias_rows(float *x, const float *bias, int rows, int dim) { } } -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, const int *token_ids, int n_tokens, float *out_embedding) { if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_embedding) return -1; - const ffwd_config_t *cfg = &model->config; - const ffwd_weights_t *w = &model->weights; + const langdet_config_t *cfg = &model->config; + const langdet_weights_t *w = &model->weights; int hidden = cfg->hidden_size; int n_heads = cfg->n_heads; int head_dim = cfg->head_dim; @@ -32,7 +32,7 @@ int ffwd_model_encode_into(const ffwd_model_t *model, if (n_tokens > cfg->max_position_embeddings - cfg->position_id_offset) { fprintf(stderr, - "ffwd: sequence length %d plus position offset %d exceeds max_position_embeddings " + "langdet: sequence length %d plus position offset %d exceeds max_position_embeddings " "%d\n", n_tokens, cfg->position_id_offset, cfg->max_position_embeddings); return -1; @@ -50,7 +50,7 @@ int ffwd_model_encode_into(const ffwd_model_t *model, for (int i = 0; i < n_tokens; i++) { int id = token_ids[i]; if (id < 0 || id >= cfg->vocab_size) { - fprintf(stderr, "ffwd: invalid token id %d at position %d\n", id, i); + fprintf(stderr, "langdet: invalid token id %d at position %d\n", id, i); return -1; } float *row = x + (size_t)i * hidden; @@ -59,12 +59,12 @@ int ffwd_model_encode_into(const ffwd_model_t *model, add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden); add_inplace(row, w->token_type_embeddings, hidden); } - layer_norm(x, x, w->ffwd_ln_w, w->ffwd_ln_b, n_tokens, hidden, eps); + layer_norm(x, x, w->langdet_ln_w, w->langdet_ln_b, n_tokens, hidden, eps); float scale = 1.0f / sqrtf((float)head_dim); for (int layer = 0; layer < cfg->n_layers; layer++) { - const ffwd_layer_t *l = &w->layers[layer]; + const langdet_layer_t *l = &w->layers[layer]; linear_nobias(q_buf, x, l->wq, n_tokens, hidden, hidden); linear_nobias(k_buf, x, l->wk, n_tokens, hidden, hidden); diff --git a/libffwd/gemm.c b/liblangdet/gemm.c index c750be1..c750be1 100644 --- a/libffwd/gemm.c +++ b/liblangdet/gemm.c diff --git a/libffwd/generic.c b/liblangdet/generic.c index 729c610..729c610 100644 --- a/libffwd/generic.c +++ b/liblangdet/generic.c diff --git a/libffwd/impl.h b/liblangdet/impl.h index f216518..f216518 100644 --- a/libffwd/impl.h +++ b/liblangdet/impl.h diff --git a/libffwd/kernels.h b/liblangdet/kernels.h index 1025602..1025602 100644 --- a/libffwd/kernels.h +++ b/liblangdet/kernels.h diff --git a/libffwd/ffwd.h b/liblangdet/langdet.h index 527e92f..9c24264 100644 --- a/libffwd/ffwd.h +++ b/liblangdet/langdet.h @@ -1,9 +1,9 @@ -#ifndef FFWD_H -#define FFWD_H +#ifndef LANGDET_H +#define LANGDET_H #include <stddef.h> -#define FFWD_MAX_LAYERS 64 +#define LANGDET_MAX_LAYERS 64 typedef struct { int hidden_size; @@ -16,34 +16,34 @@ typedef struct { int position_id_offset; /* XLM-R positions start at pad_token_id + 1. */ int type_vocab_size; float layer_norm_eps; -} ffwd_config_t; +} langdet_config_t; -typedef struct ffwd_model ffwd_model_t; -typedef struct ffwd_workspace ffwd_workspace_t; -typedef struct ffwd_tok ffwd_tok_t; +typedef struct langdet_model langdet_model_t; +typedef struct langdet_workspace langdet_workspace_t; +typedef struct langdet_tok langdet_tok_t; /* The model owns its memory-mapped F32 weights. Workspaces must be freed first. */ -ffwd_model_t *ffwd_model_load(const char *model_dir); -void ffwd_model_free(ffwd_model_t *model); -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model); +langdet_model_t *langdet_model_load(const char *model_dir); +void langdet_model_free(langdet_model_t *model); +const langdet_config_t *langdet_model_config(const langdet_model_t *model); /* A workspace belongs to one model and may serve only one inference call at a time. */ -ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model); -void ffwd_workspace_free(ffwd_workspace_t *ws); +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model); +void langdet_workspace_free(langdet_workspace_t *ws); /* Write the first token's final hidden state to out_embedding[hidden_size]. * Token IDs must include <s> and </s>, without padding. Returns 0 or -1 on error. */ -int ffwd_model_encode_into(const ffwd_model_t *model, - ffwd_workspace_t *ws, +int langdet_model_encode_into(const langdet_model_t *model, + langdet_workspace_t *ws, const int *token_ids, int n_tokens, float *out_embedding); -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen); -void ffwd_tok_free(ffwd_tok_t *t); +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen); +void langdet_tok_free(langdet_tok_t *t); /* Return malloc-owned IDs wrapped with <s> and </s>, or NULL on error. * The tokenizer owns reusable scratch and is not safe for concurrent calls. */ -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out); +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out); -#endif /* FFWD_H */ +#endif /* LANGDET_H */ diff --git a/libffwd/model.c b/liblangdet/model.c index 81073f3..146bbe7 100644 --- a/libffwd/model.c +++ b/liblangdet/model.c @@ -10,11 +10,11 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, const int64_t *shape, int ndim) { if (t->dtype != DTYPE_F32) { - fprintf(stderr, "ffwd: expected F32 tensor: %s\n", name); + fprintf(stderr, "langdet: expected F32 tensor: %s\n", name); return 0; } if (t->ndim != ndim) { - fprintf(stderr, "ffwd: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); + fprintf(stderr, "langdet: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim); return 0; } @@ -22,25 +22,25 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, for (int i = 0; i < ndim; i++) { if (t->shape[i] != shape[i]) { fprintf(stderr, - "ffwd: bad shape for %s at dim %d: got %lld, expected %lld\n", + "langdet: bad shape for %s at dim %d: got %lld, expected %lld\n", name, i, (long long)t->shape[i], (long long)shape[i]); return 0; } if (shape[i] <= 0 || (uint64_t)shape[i] > SIZE_MAX / numel) { - fprintf(stderr, "ffwd: shape too large or empty for %s\n", name); + fprintf(stderr, "langdet: shape too large or empty for %s\n", name); return 0; } numel *= (size_t)shape[i]; } size_t bytes; if (mul_size(numel, sizeof(float), &bytes) != 0 || t->data_size != bytes) { - fprintf(stderr, "ffwd: invalid data size for %s\n", name); + fprintf(stderr, "langdet: invalid data size for %s\n", name); return 0; } size_t data_start = 8 + sf->header_size; if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start || t->data_size > sf->file_size - data_start - t->data_offset) { - fprintf(stderr, "ffwd: tensor data out of bounds for %s\n", name); + fprintf(stderr, "langdet: tensor data out of bounds for %s\n", name); return 0; } return 1; @@ -51,7 +51,7 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name, safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { - fprintf(stderr, "ffwd: tensor not found: %s\n", name); + fprintf(stderr, "langdet: tensor not found: %s\n", name); return NULL; } if (!tensor_has_supported_shape(sf, t, name, shape, ndim)) @@ -59,9 +59,9 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name, return safetensors_data(sf, t); } -static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) { - const ffwd_config_t *cfg = &model->config; - ffwd_weights_t *w = &model->weights; +static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) { + const langdet_config_t *cfg = &model->config; + langdet_weights_t *w = &model->weights; char name[256]; int hidden = cfg->hidden_size, inter = cfg->intermediate_size; @@ -71,18 +71,18 @@ static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) { (int64_t[]){cfg->max_position_embeddings, hidden}, 2); w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", (int64_t[]){cfg->type_vocab_size, hidden}, 2); - w->ffwd_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); - w->ffwd_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); + w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); + w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || - !w->ffwd_ln_w || !w->ffwd_ln_b) + !w->langdet_ln_w || !w->langdet_ln_b) return -1; - w->layers = (ffwd_layer_t *)calloc(cfg->n_layers, sizeof(ffwd_layer_t)); + w->layers = (langdet_layer_t *)calloc(cfg->n_layers, sizeof(langdet_layer_t)); if (!w->layers) return -1; for (int i = 0; i < cfg->n_layers; i++) { - ffwd_layer_t *l = &w->layers[i]; + langdet_layer_t *l = &w->layers[i]; #define BERT_W(field, suffix, d0, d1) \ do { \ @@ -124,17 +124,17 @@ static int load_bert_weights(ffwd_model_t *model, multi_safetensors_t *ms) { return 0; } -ffwd_model_t *ffwd_model_load(const char *model_dir) { +langdet_model_t *langdet_model_load(const char *model_dir) { if (!model_dir) return NULL; - ffwd_model_t *model = (ffwd_model_t *)calloc(1, sizeof(ffwd_model_t)); + langdet_model_t *model = (langdet_model_t *)calloc(1, sizeof(langdet_model_t)); if (!model) return NULL; - if (ffwd_config_parse(&model->config, model_dir) != 0) + if (langdet_config_parse(&model->config, model_dir) != 0) goto fail; model->safetensors = multi_safetensors_open(model_dir); if (!model->safetensors) { - fprintf(stderr, "ffwd_model_load: failed to open safetensors in %s\n", model_dir); + fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir); goto fail; } if (load_bert_weights(model, model->safetensors) != 0) @@ -142,11 +142,11 @@ ffwd_model_t *ffwd_model_load(const char *model_dir) { return model; fail: - ffwd_model_free(model); + langdet_model_free(model); return NULL; } -void ffwd_model_free(ffwd_model_t *model) { +void langdet_model_free(langdet_model_t *model) { if (!model) return; multi_safetensors_close(model->safetensors); @@ -154,6 +154,6 @@ void ffwd_model_free(ffwd_model_t *model) { free(model); } -const ffwd_config_t *ffwd_model_config(const ffwd_model_t *model) { +const langdet_config_t *langdet_model_config(const langdet_model_t *model) { return model ? &model->config : NULL; } diff --git a/libffwd/model_blas.c b/liblangdet/model_blas.c index 409ba66..41f9237 100644 --- a/libffwd/model_blas.c +++ b/liblangdet/model_blas.c @@ -3,17 +3,17 @@ #include <stdlib.h> -ffwd_workspace_t *ffwd_workspace_new(const ffwd_model_t *model) { +langdet_workspace_t *langdet_workspace_new(const langdet_model_t *model) { if (!model) return NULL; - ffwd_workspace_t *ws = (ffwd_workspace_t *)calloc(1, sizeof(ffwd_workspace_t)); + langdet_workspace_t *ws = (langdet_workspace_t *)calloc(1, sizeof(langdet_workspace_t)); if (!ws) return NULL; ws->model = model; return ws; } -void ffwd_workspace_free(ffwd_workspace_t *ws) { +void langdet_workspace_free(langdet_workspace_t *ws) { if (!ws) return; free(ws->x); @@ -27,7 +27,7 @@ void ffwd_workspace_free(ffwd_workspace_t *ws) { free(ws); } -int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) { +int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq) { if (seq <= ws->buf_seq_cap) return 0; @@ -55,7 +55,7 @@ int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *c, int seq) { return 0; } -void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch) { +void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) { size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); if (bytes <= ws->attn_scores_bytes) return; diff --git a/libffwd/model_internal.h b/liblangdet/model_internal.h index aeea52e..73a7851 100644 --- a/libffwd/model_internal.h +++ b/liblangdet/model_internal.h @@ -1,5 +1,5 @@ -#ifndef FFWD_MODEL_INTERNAL_H -#define FFWD_MODEL_INTERNAL_H +#ifndef LANGDET_MODEL_INTERNAL_H +#define LANGDET_MODEL_INTERNAL_H #include "model_types.h" #include "alloc.h" @@ -11,7 +11,7 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, const int64_t *shape, int ndim); -int ensure_buffers(ffwd_workspace_t *ws, const ffwd_config_t *cfg, int seq); -void ensure_attention_scores(ffwd_workspace_t *ws, const int *offsets, int batch); +int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq); +void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch); -#endif /* FFWD_MODEL_INTERNAL_H */ +#endif /* LANGDET_MODEL_INTERNAL_H */ diff --git a/libffwd/model_types.h b/liblangdet/model_types.h index 447b653..2170909 100644 --- a/libffwd/model_types.h +++ b/liblangdet/model_types.h @@ -1,7 +1,7 @@ -#ifndef FFWD_MODEL_TYPES_H -#define FFWD_MODEL_TYPES_H +#ifndef LANGDET_MODEL_TYPES_H +#define LANGDET_MODEL_TYPES_H -#include "ffwd.h" +#include "langdet.h" #include "safetensors.h" typedef struct { @@ -21,26 +21,26 @@ typedef struct { const float *ffn_out_bias; const float *post_attn_norm; const float *ffn_ln_bias; -} ffwd_layer_t; +} langdet_layer_t; typedef struct { const float *embed_tokens; - ffwd_layer_t *layers; + langdet_layer_t *layers; const float *position_embeddings; const float *token_type_embeddings; - const float *ffwd_ln_w; - const float *ffwd_ln_b; -} ffwd_weights_t; + const float *langdet_ln_w; + const float *langdet_ln_b; +} langdet_weights_t; -struct ffwd_model { - ffwd_config_t config; - ffwd_weights_t weights; +struct langdet_model { + langdet_config_t config; + langdet_weights_t weights; /* All weight pointers borrow this mapping until model teardown. */ multi_safetensors_t *safetensors; }; -struct ffwd_workspace { - const ffwd_model_t *model; +struct langdet_workspace { + const langdet_model_t *model; int buf_seq_cap; float *x; float *q; @@ -53,4 +53,4 @@ struct ffwd_workspace { size_t attn_scores_bytes; }; -#endif /* FFWD_MODEL_TYPES_H */ +#endif /* LANGDET_MODEL_TYPES_H */ diff --git a/libffwd/norm.c b/liblangdet/norm.c index 59e8bdb..59e8bdb 100644 --- a/libffwd/norm.c +++ b/liblangdet/norm.c diff --git a/libffwd/safetensors.c b/liblangdet/safetensors.c index cfba76e..cfba76e 100644 --- a/libffwd/safetensors.c +++ b/liblangdet/safetensors.c diff --git a/libffwd/safetensors.h b/liblangdet/safetensors.h index dca68a7..dca68a7 100644 --- a/libffwd/safetensors.h +++ b/liblangdet/safetensors.h diff --git a/libffwd/sentencepiece.c b/liblangdet/sentencepiece.c index 6c30ba1..6c30ba1 100644 --- a/libffwd/sentencepiece.c +++ b/liblangdet/sentencepiece.c diff --git a/libffwd/sentencepiece.h b/liblangdet/sentencepiece.h index 2230538..2230538 100644 --- a/libffwd/sentencepiece.h +++ b/liblangdet/sentencepiece.h diff --git a/libffwd/threadpool.c b/liblangdet/threadpool.c index 6cb4252..6cb4252 100644 --- a/libffwd/threadpool.c +++ b/liblangdet/threadpool.c diff --git a/libffwd/threadpool.h b/liblangdet/threadpool.h index f9739a1..f9739a1 100644 --- a/libffwd/threadpool.h +++ b/liblangdet/threadpool.h diff --git a/libffwd/tokenize.c b/liblangdet/tokenize.c index 8e1000f..5e26a18 100644 --- a/libffwd/tokenize.c +++ b/liblangdet/tokenize.c @@ -1,4 +1,4 @@ -#include "ffwd.h" +#include "langdet.h" #include "sentencepiece.h" #include <limits.h> @@ -6,7 +6,7 @@ #include <stdlib.h> #include <string.h> -struct ffwd_tok { +struct langdet_tok { tok_spm_t *spm; tok_spm_workspace_t *workspace; int cls_id; @@ -18,12 +18,12 @@ static void set_err(char *err, size_t errlen, const char *msg) { snprintf(err, errlen, "%s", msg); } -ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { +langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) { if (!model_dir) { set_err(err, errlen, "model_dir is NULL"); return NULL; } - ffwd_tok_t *t = calloc(1, sizeof(*t)); + langdet_tok_t *t = calloc(1, sizeof(*t)); if (!t) { set_err(err, errlen, "out of memory"); return NULL; @@ -47,11 +47,11 @@ ffwd_tok_t *ffwd_tok_open(const char *model_dir, char *err, size_t errlen) { return t; fail: - ffwd_tok_free(t); + langdet_tok_free(t); return NULL; } -void ffwd_tok_free(ffwd_tok_t *t) { +void langdet_tok_free(langdet_tok_t *t) { if (!t) return; tok_spm_workspace_free(t->workspace); @@ -59,7 +59,7 @@ void ffwd_tok_free(ffwd_tok_t *t) { free(t); } -int *ffwd_tokenize(ffwd_tok_t *t, const char *text, int *n_out) { +int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { if (n_out) *n_out = 0; if (!t || !text) |
