aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile8
-rw-r--r--README.md17
-rw-r--r--liblangdet/attn.c246
-rw-r--r--liblangdet/forward.c8
-rw-r--r--liblangdet/kernels.h26
-rw-r--r--liblangdet/langdet.c4
-rw-r--r--liblangdet/model.c91
-rw-r--r--liblangdet/model_blas.c4
-rw-r--r--liblangdet/model_internal.h2
-rw-r--r--liblangdet/model_types.h2
-rw-r--r--liblangdet/safetensors.c466
-rw-r--r--liblangdet/safetensors.h27
-rw-r--r--liblangdet/sentencepiece.c229
-rw-r--r--liblangdet/sentencepiece.h39
-rw-r--r--liblangdet/tokenize.c12
-rw-r--r--tools/test_safetensors.c74
-rw-r--r--tools/verify.py30
17 files changed, 379 insertions, 906 deletions
diff --git a/Makefile b/Makefile
index 9697934..7ea80f8 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ CPU_SOURCES = langdet.c model.c model_blas.c forward.c alloc.c config.c \
LIB_SOURCES = $(addprefix liblangdet/,$(CPU_SOURCES)) deps/yyjson/yyjson.c
LIB_OBJECTS = $(LIB_SOURCES:%.c=build/%.o)
OBJECTS = build/langdetect.o $(LIB_OBJECTS)
-TEST_OBJECTS = build/tools/test_api.o build/tools/token_ids.o
+TEST_OBJECTS = build/tools/test_api.o build/tools/token_ids.o build/tools/test_safetensors.o
.PHONY: all clean test
all: langdetect liblangdet.a
@@ -32,7 +32,11 @@ build/test_api: build/tools/test_api.o liblangdet.a
build/token_ids: build/tools/token_ids.o liblangdet.a
$(CC) $(LDFLAGS) -o $@ $< liblangdet.a $(LDLIBS)
-test: all build/test_api build/token_ids
+build/test_safetensors: build/tools/test_safetensors.o liblangdet.a
+ $(CC) $(LDFLAGS) -o $@ $< liblangdet.a $(LDLIBS)
+
+test: all build/test_api build/token_ids build/test_safetensors
+ ./build/test_safetensors
./build/test_api "$(MODEL)"
UV_CACHE_DIR="$(CURDIR)/.uv-cache" UV_PYTHON_DOWNLOADS=never \
HF_HOME="$(CURDIR)/.hf-cache" HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \
diff --git a/README.md b/README.md
index 23a6612..0735571 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,9 @@ printf '%s\n' 'Bonjour tout le monde.' 'これは日本語です。' | ./langdet
Run these commands from this directory. The default model path is `./model`;
use `-d DIRECTORY` for a different path.
+The Makefile currently targets macOS ARM64. The source includes Linux/OpenBLAS
+and x86 paths, but Linux still needs build configuration and testing.
+
The model is loaded once per process. Each text argument, or each stdin line
when there are no text arguments, produces one JSON line in input order:
@@ -44,6 +47,10 @@ The encoder uses first-token pooling without L2 normalization.
The classifier applies a 768-to-768 linear layer, `tanh`, a 768-to-20 linear layer,
and stable softmax. Dropout is disabled during inference.
+The native loader requires `config.json`, one `model.safetensors` file, and
+`sentencepiece.bpe.model`. Sharded weights and alternate tokenizer files are not
+supported. The safetensors and configuration readers both use `yyjson`.
+
The CPU encoder and tokenizer are in `liblangdet/`; the JSON parser is in
`deps/yyjson/`. Their license files are included. The build uses only local source
files and the macOS SDK, with no dependency on another repository.
@@ -128,15 +135,19 @@ The target builds and runs:
- `tools/test_api.c`: repeated predictions, score normalization, result ownership,
empty input, truncation, invalid arguments, error buffers, and detector lifecycle.
+- `tools/test_safetensors.c`: single-file tensor loading, metadata, lookup, and
+ rejection of malformed shapes, offsets, and JSON headers.
- `tools/token_ids.c`: a test-only adapter that returns the native token IDs.
- `tools/verify.py`: exact token-ID comparison and all 20 language scores against
- Transformers on 28 inputs. These cover the 20 languages, empty and whitespace
+ Transformers on 31 inputs. These cover the 20 languages, empty and whitespace
input, Unicode normalization, multiple lines, and lengths below, at, and above
- the 512-token limit. Scores must differ by no more than `1e-4` in absolute value,
+ the 128-token BLAS threshold and the 512-token limit. Scores must differ by no more than `1e-4` in absolute value,
and the predicted labels must match, with both one and four CPU kernel threads.
The Python checks also verify stdin behavior, invalid CLI arguments, NUL rejection,
and failures for missing or invalid configuration, weights, and tokenizer files.
+They also check rejection of sharded weights, alternate tokenizer filenames, and
+BERT model configuration.
No Python package is linked into or required by the native library.
## Verification status
@@ -151,6 +162,6 @@ No Python package is linked into or required by the native library.
- Standalone C and C++ applications link using only `langdet.h` and the static
archive. Prediction results, error returns, and detector lifecycle checks pass.
- The downloaded checkpoint files match Hugging Face's sizes and hashes.
-- The reference comparisons pass on the 28 test inputs: token IDs match exactly,
+- The reference comparisons pass on the 31 test inputs: token IDs match exactly,
labels agree, and probability differences stay below the `1e-4` tolerance.
These regression checks are not an accuracy benchmark or exhaustive tokenizer test.
diff --git a/liblangdet/attn.c b/liblangdet/attn.c
index ac0f4d1..42d33de 100644
--- a/liblangdet/attn.c
+++ b/liblangdet/attn.c
@@ -12,112 +12,82 @@
# endif
#endif
-static inline float dot_f32_fast(const float *a, const float *b, int n) {
- return dot_f32_impl(a, b, n);
-}
-
-/* dst = dst * scale */
-static inline void vec_scale_inplace(float *dst, float scale, int n) {
- vec_scale_inplace_impl(dst, scale, n);
-}
-
-/* dst += alpha * src */
-static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
- vec_axpy_inplace_impl(dst, src, alpha, n);
-}
-
-/* dst = dst * correction + src */
-static inline void vec_scale_add(float *dst, const float *src, float correction, int n) {
- vec_scale_add_impl(dst, src, correction, n);
-}
-
-#define PACKED_ATTN_QUERY_TILE 32
-#define PACKED_ATTN_BLAS_MIN_SEQ 128
+#define ATTN_QUERY_TILE 32
+#define ATTN_BLAS_MIN_SEQ 128
-static void bidirectional_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
+static void attention_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
+ int hidden = n_heads * head_dim;
for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+ const float *q_row = Q + (size_t)i * hidden + h * head_dim;
+ float *o_row = out + (size_t)i * hidden + h * head_dim;
float max_score = -1e30f;
float sum_exp = 0.0f;
for (int d = 0; d < head_dim; d++)
o_row[d] = 0.0f;
- for (int j = start; j < end; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+ for (int j = 0; j < seq; j++) {
+ const float *k_row = K + (size_t)j * hidden + h * head_dim;
+ const float *v_row = V + (size_t)j * hidden + h * head_dim;
- float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
+ float score = dot_f32_impl(q_row, k_row, head_dim) * scale;
if (score > max_score) {
float correction = expf(max_score - score);
sum_exp = sum_exp * correction + 1.0f;
- vec_scale_add(o_row, v_row, correction, head_dim);
+ vec_scale_add_impl(o_row, v_row, correction, head_dim);
max_score = score;
} else {
float wt = expf(score - max_score);
sum_exp += wt;
- vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ vec_axpy_inplace_impl(o_row, v_row, wt, head_dim);
}
}
if (sum_exp > 0.0f) {
float inv_sum = 1.0f / sum_exp;
- vec_scale_inplace(o_row, inv_sum, head_dim);
+ vec_scale_inplace_impl(o_row, inv_sum, head_dim);
}
}
}
#ifdef USE_BLAS
-static void bidirectional_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
+static void attention_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
+ int hidden = n_heads * head_dim;
int rows = query_end - query_start;
- int keys = end - start;
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
+ const float *q = Q + (size_t)query_start * hidden + h * head_dim;
+ const float *k = K + h * head_dim;
+ const float *v = V + h * head_dim;
+ float *o = out + (size_t)query_start * hidden + h * head_dim;
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
- kv_hidden, 0.0f, scores, keys);
- softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, seq, head_dim, scale, q, hidden, k,
+ hidden, 0.0f, scores, seq);
+ softmax(scores, rows, seq);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, seq, 1.0f, scores, seq,
+ v, hidden, 0.0f, o, hidden);
}
#endif
@@ -126,117 +96,85 @@ typedef struct {
const float *Q;
const float *K;
const float *V;
- const int *offsets;
- int batch;
- int n_heads, n_kv_heads;
+ int seq;
+ int n_heads;
int head_dim;
float scale;
float *scores;
- int max_seq;
-} packed_gqa_attn_task_t;
+} attention_task_t;
-static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+static void attention_worker(int tid, int n_threads, void *arg) {
+ attention_task_t *t = (attention_task_t *)arg;
int task_id = 0;
#ifdef USE_BLAS
- float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
+ float *scores = t->scores ? t->scores + (size_t)tid * ATTN_QUERY_TILE * t->seq : NULL;
#endif
for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
+ for (int q0 = 0; q0 < t->seq; q0 += ATTN_QUERY_TILE) {
+ int q1 = q0 + ATTN_QUERY_TILE;
+ if (q1 > t->seq)
+ q1 = t->seq;
+ if (task_id++ % n_threads != tid)
+ continue;
#ifdef USE_BLAS
- if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
- bidirectional_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end,
- t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
+ if (scores && t->seq >= ATTN_BLAS_MIN_SEQ) {
+ attention_blas_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads,
+ t->head_dim, t->scale, h, q0, q1, scores);
+ continue;
}
+#endif
+ attention_online_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads,
+ t->head_dim, t->scale, h, q0, q1);
}
}
}
-size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
+size_t attention_scratch_bytes(int seq) {
#ifdef USE_BLAS
- if (!offsets || batch <= 0)
+ if (seq < ATTN_BLAS_MIN_SEQ)
return 0;
-
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- if (len > max_seq)
- max_seq = len;
- }
- if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ)
+ if ((size_t)seq > SIZE_MAX / (sizeof(float) * ATTN_QUERY_TILE * (size_t)tp_num_threads()))
return 0;
- if ((size_t)max_seq >
- SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads()))
- return 0;
- return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
+ return (size_t)tp_num_threads() * ATTN_QUERY_TILE * (size_t)seq * sizeof(float);
#else
- (void)offsets;
- (void)batch;
+ (void)seq;
return 0;
#endif
}
-void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * len * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
+void attention_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
+ long long qk_work = (long long)seq * seq * n_heads;
float *scores = NULL;
#ifdef USE_BLAS
- size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ size_t required = attention_scratch_bytes(seq);
if (required != 0 && scratch && scratch_bytes >= required)
scores = scratch;
#else
(void)scratch;
(void)scratch_bytes;
#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
+ attention_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .seq = seq,
+ .n_heads = n_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores};
if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
- parallel_for(packed_gqa_attn_worker, &task);
+ parallel_for(attention_worker, &task);
} else {
- packed_gqa_attn_worker(0, 1, &task);
+ attention_worker(0, 1, &task);
}
}
diff --git a/liblangdet/forward.c b/liblangdet/forward.c
index 14b1327..cede404 100644
--- a/liblangdet/forward.c
+++ b/liblangdet/forward.c
@@ -40,8 +40,7 @@ int langdet_model_encode_into(const langdet_model_t *model,
if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter ||
ensure_buffers(ws, cfg, n_tokens) != 0)
return -1;
- int offsets[2] = {0, n_tokens};
- ensure_attention_scores(ws, offsets, 1);
+ ensure_attention_scores(ws, n_tokens);
float *x = ws->x;
float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v;
@@ -73,9 +72,8 @@ int langdet_model_encode_into(const langdet_model_t *model,
add_bias_rows(k_buf, l->k_bias, n_tokens, hidden);
add_bias_rows(v_buf, l->v_bias, n_tokens, hidden);
- bidirectional_gqa_attention_packed_with_scratch(
- attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale,
- ws->attn_scores, ws->attn_scores_bytes);
+ attention_with_scratch(attn_out, q_buf, k_buf, v_buf, n_tokens, n_heads, head_dim, scale,
+ ws->attn_scores, ws->attn_scores_bytes);
linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden);
add_bias_rows(proj_out, l->o_bias, n_tokens, hidden);
diff --git a/liblangdet/kernels.h b/liblangdet/kernels.h
index 1025602..95f5df0 100644
--- a/liblangdet/kernels.h
+++ b/liblangdet/kernels.h
@@ -23,20 +23,18 @@ void layer_norm(float *out,
void softmax(float *x, int rows, int cols);
void gelu_inplace(float *x, int n);
-/* Attention is block-diagonal: each token attends only within its offsets span.
+/* Bidirectional attention over one unpadded sequence with equal Q/K/V head counts.
* Missing scratch uses the online-softmax path instead of tiled BLAS. */
-size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
-void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+size_t attention_scratch_bytes(int seq);
+void attention_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
#endif /* KERNELS_H */
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c
index 9410ba2..0fe5433 100644
--- a/liblangdet/langdet.c
+++ b/liblangdet/langdet.c
@@ -36,8 +36,8 @@ static int set_error(char *err, size_t errlen, const char *format, ...) {
static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape,
int ndim, char *err, size_t errlen) {
- safetensors_file_t *file = NULL;
- const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
+ const safetensors_file_t *file = d->model->safetensors;
+ const safetensor_t *tensor = safetensors_find(file, name);
if (!tensor) {
set_error(err, errlen, "Missing classifier tensor: %s", name);
return NULL;
diff --git a/liblangdet/model.c b/liblangdet/model.c
index 146bbe7..53f3692 100644
--- a/liblangdet/model.c
+++ b/liblangdet/model.c
@@ -46,10 +46,9 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static const float *load_weight(multi_safetensors_t *ms, const char *name,
+static const float *load_weight(const safetensors_file_t *sf, const char *name,
const int64_t *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = safetensors_find(sf, name);
if (!t) {
fprintf(stderr, "langdet: tensor not found: %s\n", name);
return NULL;
@@ -59,20 +58,20 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name,
return safetensors_data(sf, t);
}
-static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
+static int load_xlm_roberta_weights(langdet_model_t *model, const safetensors_file_t *sf) {
const langdet_config_t *cfg = &model->config;
langdet_weights_t *w = &model->weights;
char name[256];
int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
- w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight",
+ w->embed_tokens = load_weight(sf, "roberta.embeddings.word_embeddings.weight",
(int64_t[]){cfg->vocab_size, hidden}, 2);
- w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight",
+ w->position_embeddings = load_weight(sf, "roberta.embeddings.position_embeddings.weight",
(int64_t[]){cfg->max_position_embeddings, hidden}, 2);
- w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
+ w->token_type_embeddings = load_weight(sf, "roberta.embeddings.token_type_embeddings.weight",
(int64_t[]){cfg->type_vocab_size, hidden}, 2);
- w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
- w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
+ w->langdet_ln_w = load_weight(sf, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
+ w->langdet_ln_b = load_weight(sf, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
!w->langdet_ln_w || !w->langdet_ln_b)
return -1;
@@ -84,42 +83,42 @@ static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
for (int i = 0; i < cfg->n_layers; i++) {
langdet_layer_t *l = &w->layers[i];
-#define BERT_W(field, suffix, d0, d1) \
- do { \
- const int64_t e[2] = {(d0), (d1)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 2); \
- if (!l->field) \
- return -1; \
+#define LOAD_MATRIX(field, suffix, d0, d1) \
+ do { \
+ const int64_t e[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(sf, name, e, 2); \
+ if (!l->field) \
+ return -1; \
} while (0)
-#define BERT_V(field, suffix, d0) \
- do { \
- const int64_t e[1] = {(d0)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 1); \
- if (!l->field) \
- return -1; \
+#define LOAD_VECTOR(field, suffix, d0) \
+ do { \
+ const int64_t e[1] = {(d0)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(sf, name, e, 1); \
+ if (!l->field) \
+ return -1; \
} while (0)
- BERT_W(wq, "attention.self.query.weight", hidden, hidden);
- BERT_V(q_bias, "attention.self.query.bias", hidden);
- BERT_W(wk, "attention.self.key.weight", hidden, hidden);
- BERT_V(k_bias, "attention.self.key.bias", hidden);
- BERT_W(wv, "attention.self.value.weight", hidden, hidden);
- BERT_V(v_bias, "attention.self.value.bias", hidden);
- BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
- BERT_V(o_bias, "attention.output.dense.bias", hidden);
- BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
- BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
- BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
- BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
- BERT_W(down_proj, "output.dense.weight", hidden, inter);
- BERT_V(ffn_out_bias, "output.dense.bias", hidden);
- BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
- BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
+ LOAD_MATRIX(wq, "attention.self.query.weight", hidden, hidden);
+ LOAD_VECTOR(q_bias, "attention.self.query.bias", hidden);
+ LOAD_MATRIX(wk, "attention.self.key.weight", hidden, hidden);
+ LOAD_VECTOR(k_bias, "attention.self.key.bias", hidden);
+ LOAD_MATRIX(wv, "attention.self.value.weight", hidden, hidden);
+ LOAD_VECTOR(v_bias, "attention.self.value.bias", hidden);
+ LOAD_MATRIX(wo, "attention.output.dense.weight", hidden, hidden);
+ LOAD_VECTOR(o_bias, "attention.output.dense.bias", hidden);
+ LOAD_VECTOR(input_norm, "attention.output.LayerNorm.weight", hidden);
+ LOAD_VECTOR(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
+ LOAD_MATRIX(up_proj, "intermediate.dense.weight", inter, hidden);
+ LOAD_VECTOR(ffn_inter_bias, "intermediate.dense.bias", inter);
+ LOAD_MATRIX(down_proj, "output.dense.weight", hidden, inter);
+ LOAD_VECTOR(ffn_out_bias, "output.dense.bias", hidden);
+ LOAD_VECTOR(post_attn_norm, "output.LayerNorm.weight", hidden);
+ LOAD_VECTOR(ffn_ln_bias, "output.LayerNorm.bias", hidden);
-#undef BERT_W
-#undef BERT_V
+#undef LOAD_MATRIX
+#undef LOAD_VECTOR
}
return 0;
}
@@ -132,12 +131,16 @@ langdet_model_t *langdet_model_load(const char *model_dir) {
return NULL;
if (langdet_config_parse(&model->config, model_dir) != 0)
goto fail;
- model->safetensors = multi_safetensors_open(model_dir);
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
+ if (length < 0 || (size_t)length >= sizeof(path))
+ goto fail;
+ model->safetensors = safetensors_open(path);
if (!model->safetensors) {
fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir);
goto fail;
}
- if (load_bert_weights(model, model->safetensors) != 0)
+ if (load_xlm_roberta_weights(model, model->safetensors) != 0)
goto fail;
return model;
@@ -149,7 +152,7 @@ fail:
void langdet_model_free(langdet_model_t *model) {
if (!model)
return;
- multi_safetensors_close(model->safetensors);
+ safetensors_close(model->safetensors);
free(model->weights.layers);
free(model);
}
diff --git a/liblangdet/model_blas.c b/liblangdet/model_blas.c
index 41f9237..d26db32 100644
--- a/liblangdet/model_blas.c
+++ b/liblangdet/model_blas.c
@@ -55,8 +55,8 @@ int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq)
return 0;
}
-void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) {
- size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+void ensure_attention_scores(langdet_workspace_t *ws, int seq) {
+ size_t bytes = attention_scratch_bytes(seq);
if (bytes <= ws->attn_scores_bytes)
return;
diff --git a/liblangdet/model_internal.h b/liblangdet/model_internal.h
index 73a7851..ed7f237 100644
--- a/liblangdet/model_internal.h
+++ b/liblangdet/model_internal.h
@@ -12,6 +12,6 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
int ndim);
int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq);
-void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch);
+void ensure_attention_scores(langdet_workspace_t *ws, int seq);
#endif /* LANGDET_MODEL_INTERNAL_H */
diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h
index 31d101a..aa46d39 100644
--- a/liblangdet/model_types.h
+++ b/liblangdet/model_types.h
@@ -36,7 +36,7 @@ struct langdet_model {
langdet_config_t config;
langdet_weights_t weights;
/* All weight pointers borrow this mapping until model teardown. */
- multi_safetensors_t *safetensors;
+ safetensors_file_t *safetensors;
};
struct langdet_workspace {
diff --git a/liblangdet/safetensors.c b/liblangdet/safetensors.c
index cfba76e..546f51d 100644
--- a/liblangdet/safetensors.c
+++ b/liblangdet/safetensors.c
@@ -1,267 +1,68 @@
#include "safetensors.h"
-#include <dirent.h>
+#include <yyjson.h>
+
#include <fcntl.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
-/* Minimal JSON parser for safetensors header */
-/* TODO: consider replacing with yyjson */
-
-static void skip_whitespace(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
- (*p)++;
-}
-
-static int parse_string(const char **p, char *out, size_t max_len) {
- skip_whitespace(p);
- if (**p != '"')
- return -1;
- (*p)++;
- size_t i = 0;
- while (**p && **p != '"' && i < max_len - 1) {
- if (**p == '\\') {
- (*p)++;
- if (**p == 'n')
- out[i++] = '\n';
- else if (**p == 't')
- out[i++] = '\t';
- else if (**p == '"')
- out[i++] = '"';
- else if (**p == '\\')
- out[i++] = '\\';
- else
- out[i++] = **p;
- } else {
- out[i++] = **p;
- }
- (*p)++;
- }
- out[i] = '\0';
- if (**p != '"')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int parse_int(const char **p, int64_t *out) {
- skip_whitespace(p);
- int64_t val = 0;
- int neg = 0;
- if (**p == '-') {
- neg = 1;
- (*p)++;
- }
- if (**p < '0' || **p > '9')
- return -1;
- while (**p >= '0' && **p <= '9') {
- val = val * 10 + (**p - '0');
- (*p)++;
- }
- *out = neg ? -val : val;
- return 0;
-}
-
-static safetensor_dtype_t parse_dtype(const char *s) {
- if (strcmp(s, "F32") == 0)
- return DTYPE_F32;
- if (strcmp(s, "F16") == 0)
- return DTYPE_F16;
- if (strcmp(s, "BF16") == 0)
- return DTYPE_BF16;
- if (strcmp(s, "I32") == 0)
- return DTYPE_I32;
- if (strcmp(s, "I64") == 0)
- return DTYPE_I64;
- if (strcmp(s, "BOOL") == 0)
- return DTYPE_BOOL;
- return DTYPE_UNKNOWN;
-}
-
-static int parse_tensor_entry(const char **p, safetensor_t *t) {
- skip_whitespace(p);
- if (**p != '{')
- return -1;
- (*p)++;
-
- t->dtype = DTYPE_UNKNOWN;
- t->ndim = 0;
- t->data_offset = 0;
- t->data_size = 0;
-
- while (**p && **p != '}') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
-
- char key[64];
- if (parse_string(p, key, sizeof(key)) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ':')
- return -1;
- (*p)++;
- skip_whitespace(p);
-
- if (strcmp(key, "dtype") == 0) {
- char dtype_str[32];
- if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0)
- return -1;
- t->dtype = parse_dtype(dtype_str);
- } else if (strcmp(key, "shape") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- t->ndim = 0;
- while (**p && **p != ']') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
- if (t->ndim >= 8)
- return -1;
- if (parse_int(p, &t->shape[t->ndim]) != 0)
- return -1;
- t->ndim++;
- }
- if (**p != ']')
- return -1;
- (*p)++;
- } else if (strcmp(key, "data_offsets") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t start_i;
- if (parse_int(p, &start_i) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ',')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t end_i;
- if (parse_int(p, &end_i) != 0)
- return -1;
- if (start_i < 0 || end_i < start_i)
- return -1;
- t->data_offset = (size_t)start_i;
- t->data_size = (size_t)(end_i - start_i);
- skip_whitespace(p);
- if (**p != ']')
- return -1;
- (*p)++;
- } else {
- /* Skip unknown value */
- if (**p == '"') {
- (*p)++;
- while (**p && **p != '"') {
- if (**p == '\\')
- (*p)++;
- if (**p)
- (*p)++;
- }
- if (**p == '"')
- (*p)++;
- } else if (**p == '[') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '[')
- depth++;
- else if (**p == ']')
- depth--;
- (*p)++;
- }
- } else if (**p == '{') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '{')
- depth++;
- else if (**p == '}')
- depth--;
- (*p)++;
- }
- } else {
- while (**p && **p != ',' && **p != '}')
- (*p)++;
- }
- }
- }
- if (**p != '}')
- return -1;
- (*p)++;
- return 0;
-}
-
static int parse_header(safetensors_file_t *sf) {
- const char *p = sf->header_json;
- skip_whitespace(&p);
- if (*p != '{')
+ yyjson_doc *doc = yyjson_read((char *)sf->data + 8, sf->header_size, 0);
+ if (!doc)
return -1;
- p++;
-
- sf->num_tensors = 0;
-
- while (*p && *p != '}') {
- skip_whitespace(&p);
- if (*p == ',') {
- p++;
- continue;
- }
- if (*p == '}')
- break;
+ yyjson_val *root = yyjson_doc_get_root(doc);
+ if (!yyjson_is_obj(root))
+ goto fail;
- char name[256];
- if (parse_string(&p, name, sizeof(name)) != 0)
- return -1;
- skip_whitespace(&p);
- if (*p != ':')
- return -1;
- p++;
-
- if (strcmp(name, "__metadata__") == 0) {
- skip_whitespace(&p);
- if (*p == '{') {
- int depth = 1;
- p++;
- while (*p && depth > 0) {
- if (*p == '{')
- depth++;
- else if (*p == '}')
- depth--;
- p++;
- }
- }
+ size_t index, count;
+ yyjson_val *key, *entry;
+ yyjson_obj_foreach(root, index, count, key, entry) {
+ if (yyjson_equals_str(key, "__metadata__"))
continue;
- }
+ if (!yyjson_is_obj(entry) || sf->num_tensors >= SAFETENSORS_MAX_TENSORS)
+ goto fail;
+ safetensor_t *tensor = &sf->tensors[sf->num_tensors];
+ size_t name_len = yyjson_get_len(key);
+ if (name_len >= sizeof(tensor->name))
+ goto fail;
+ memcpy(tensor->name, yyjson_get_str(key), name_len + 1);
- if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) {
- fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n",
- sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS);
- return -1;
+ yyjson_val *dtype = yyjson_obj_get(entry, "dtype");
+ yyjson_val *shape = yyjson_obj_get(entry, "shape");
+ yyjson_val *offsets = yyjson_obj_get(entry, "data_offsets");
+ if (!yyjson_is_str(dtype) || !yyjson_is_arr(shape) ||
+ yyjson_arr_size(shape) > sizeof(tensor->shape) / sizeof(tensor->shape[0]) ||
+ !yyjson_is_arr(offsets) || yyjson_arr_size(offsets) != 2)
+ goto fail;
+ tensor->dtype = yyjson_equals_str(dtype, "F32") ? DTYPE_F32 : DTYPE_UNKNOWN;
+ tensor->ndim = (int)yyjson_arr_size(shape);
+ for (int i = 0; i < tensor->ndim; i++) {
+ yyjson_val *dim = yyjson_arr_get(shape, (size_t)i);
+ if (!yyjson_is_uint(dim) || yyjson_get_uint(dim) > INT64_MAX)
+ goto fail;
+ tensor->shape[i] = (int64_t)yyjson_get_uint(dim);
}
-
- safetensor_t *t = &sf->tensors[sf->num_tensors];
- snprintf(t->name, sizeof(t->name), "%s", name);
- if (parse_tensor_entry(&p, t) != 0)
- return -1;
+ yyjson_val *start = yyjson_arr_get(offsets, 0);
+ yyjson_val *end = yyjson_arr_get(offsets, 1);
+ if (!yyjson_is_uint(start) || !yyjson_is_uint(end))
+ goto fail;
+ uint64_t first = yyjson_get_uint(start), last = yyjson_get_uint(end);
+ if (first > last || last > sf->file_size - 8 - sf->header_size)
+ goto fail;
+ tensor->data_offset = (size_t)first;
+ tensor->data_size = (size_t)(last - first);
sf->num_tensors++;
}
- if (*p != '}')
- return -1;
+ yyjson_doc_free(doc);
return 0;
-}
-/* Single file operations */
+fail:
+ yyjson_doc_free(doc);
+ return -1;
+}
safetensors_file_t *safetensors_open(const char *path) {
int fd = open(path, O_RDONLY);
@@ -269,17 +70,12 @@ safetensors_file_t *safetensors_open(const char *path) {
return NULL;
struct stat st;
- if (fstat(fd, &st) < 0) {
+ if (fstat(fd, &st) < 0 || st.st_size < 8) {
close(fd);
return NULL;
}
size_t file_size = (size_t)st.st_size;
- if (file_size < 8) {
- close(fd);
- return NULL;
- }
-
void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
if (data == MAP_FAILED)
@@ -298,24 +94,13 @@ safetensors_file_t *safetensors_open(const char *path) {
return NULL;
}
- sf->path = strdup(path);
sf->data = data;
sf->file_size = file_size;
sf->header_size = (size_t)header_size;
-
- sf->header_json = malloc(header_size + 1);
- if (!sf->header_json) {
- safetensors_close(sf);
- return NULL;
- }
- memcpy(sf->header_json, (char *)data + 8, header_size);
- sf->header_json[header_size] = '\0';
-
if (parse_header(sf) != 0) {
safetensors_close(sf);
return NULL;
}
-
return sf;
}
@@ -324,160 +109,17 @@ void safetensors_close(safetensors_file_t *sf) {
return;
if (sf->data)
munmap(sf->data, sf->file_size);
- free(sf->path);
- free(sf->header_json);
free(sf);
}
-const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
- return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
-}
-
-/* Multi-shard operations */
-
-static int cmp_shard_names(const void *a, const void *b) {
- const char *sa = (const char *)a;
- const char *sb = (const char *)b;
- return strcmp(sa, sb);
-}
-
-static int parse_shard_name(const char *name, int *idx, int *total) {
- int i = 0, n = 0, consumed = 0;
- if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2)
- return -1;
- if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n)
- return -1;
- *idx = i;
- *total = n;
- return 0;
-}
-
-multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
- multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
- if (!ms)
- return NULL;
-
- char path[4096];
-
- /* Try single file first */
- snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
- safetensors_file_t *sf = safetensors_open(path);
- if (sf) {
- ms->shards[0] = sf;
- ms->num_shards = 1;
- return ms;
- }
-
- /* Scan directory for shard files */
- DIR *dir = opendir(model_dir);
- if (!dir) {
- free(ms);
- return NULL;
- }
-
- struct dirent *entry;
- char shard_names[SAFETENSORS_MAX_SHARDS][256];
- int seen[SAFETENSORS_MAX_SHARDS] = {0};
- int n_shards = 0;
- int expected_total = 0;
-
- while ((entry = readdir(dir)) != NULL) {
- if (strncmp(entry->d_name, "model-", 6) == 0 &&
- strstr(entry->d_name, ".safetensors") != NULL) {
- int shard_idx = 0, shard_total = 0;
- if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
- fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (shard_total > SAFETENSORS_MAX_SHARDS) {
- fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total,
- SAFETENSORS_MAX_SHARDS);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (expected_total == 0) {
- expected_total = shard_total;
- } else if (expected_total != shard_total) {
- fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n",
- entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (seen[shard_idx - 1]) {
- fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx);
- closedir(dir);
- free(ms);
- return NULL;
- }
- seen[shard_idx - 1] = 1;
-
- snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name);
- n_shards++;
- }
- }
- closedir(dir);
-
- if (n_shards == 0) {
- fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir);
- free(ms);
- return NULL;
- }
- if (expected_total != n_shards) {
- fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n",
- expected_total, n_shards, model_dir);
- free(ms);
- return NULL;
- }
- for (int i = 0; i < expected_total; i++) {
- if (!seen[i]) {
- fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1,
- expected_total, model_dir);
- free(ms);
- return NULL;
- }
- }
-
- /* Sort shard names to ensure consistent ordering */
- qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names);
-
- /* Open each shard */
- for (int i = 0; i < n_shards; i++) {
- snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]);
- ms->shards[i] = safetensors_open(path);
- if (!ms->shards[i]) {
- fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path);
- multi_safetensors_close(ms);
- return NULL;
- }
- }
- ms->num_shards = n_shards;
- return ms;
-}
-
-void multi_safetensors_close(multi_safetensors_t *ms) {
- if (!ms)
- return;
- for (int i = 0; i < ms->num_shards; i++) {
- safetensors_close(ms->shards[i]);
+const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name) {
+ for (int i = 0; i < sf->num_tensors; i++) {
+ if (strcmp(sf->tensors[i].name, name) == 0)
+ return &sf->tensors[i];
}
- free(ms);
+ return NULL;
}
-const safetensor_t *
-multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) {
- for (int s = 0; s < ms->num_shards; s++) {
- safetensors_file_t *sf = ms->shards[s];
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0) {
- if (out_sf)
- *out_sf = sf;
- return &sf->tensors[i];
- }
- }
- }
- return NULL;
+const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
+ return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
}
diff --git a/liblangdet/safetensors.h b/liblangdet/safetensors.h
index dca68a7..99f1cfb 100644
--- a/liblangdet/safetensors.h
+++ b/liblangdet/safetensors.h
@@ -1,7 +1,3 @@
-/*
- * Safetensors file format reader with multi-shard support.
- */
-
#ifndef SAFETENSORS_H
#define SAFETENSORS_H
@@ -9,15 +5,10 @@
#include <stdint.h>
#define SAFETENSORS_MAX_TENSORS 1024
-#define SAFETENSORS_MAX_SHARDS 8
+/* Unused checkpoint entries may have non-F32 dtypes. */
typedef enum {
DTYPE_F32 = 0,
- DTYPE_F16 = 1,
- DTYPE_BF16 = 2,
- DTYPE_I32 = 3,
- DTYPE_I64 = 4,
- DTYPE_BOOL = 5,
DTYPE_UNKNOWN = -1
} safetensor_dtype_t;
@@ -31,32 +22,18 @@ typedef struct {
} safetensor_t;
typedef struct {
- char *path;
void *data;
size_t file_size;
size_t header_size;
- char *header_json;
int num_tensors;
safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
} safetensors_file_t;
-/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
-typedef struct {
- safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
- int num_shards;
-} multi_safetensors_t;
-
/* Open a single safetensors file (memory-mapped) */
safetensors_file_t *safetensors_open(const char *path);
void safetensors_close(safetensors_file_t *sf);
-/* Open model from directory (auto-detects single file or multi-shard) */
-multi_safetensors_t *multi_safetensors_open(const char *model_dir);
-void multi_safetensors_close(multi_safetensors_t *ms);
-
-/* Find a tensor by name across all shards */
-const safetensor_t *
-multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf);
+const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name);
/* Get raw pointer to tensor data (within mmap'd region) */
const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c
index 33efab8..e60e913 100644
--- a/liblangdet/sentencepiece.c
+++ b/liblangdet/sentencepiece.c
@@ -10,7 +10,6 @@
#include "sentencepiece.h"
-#include <errno.h>
#include <float.h>
#include <limits.h>
#include <stdio.h>
@@ -22,9 +21,6 @@ enum {
SP_PIECE_NORMAL = 1,
SP_PIECE_UNKNOWN = 2,
SP_PIECE_CONTROL = 3,
- SP_PIECE_USER_DEFINED = 4,
- SP_PIECE_UNUSED = 5,
- SP_PIECE_BYTE = 6,
};
typedef struct {
@@ -96,11 +92,6 @@ static int map_get(const sp_entry_t *map, int cap, const char *key, int len) {
return -1;
}
-static int is_tokenizable_type(int type) {
- return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED ||
- type == SP_PIECE_BYTE;
-}
-
static int reserve_chars(char **buf, size_t *cap, size_t need) {
if (need <= *cap)
return 0;
@@ -302,12 +293,14 @@ static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_
goto fail;
}
}
- if (!piece)
+ /* XLM-R stores <unk>, <s>, and </s> first, followed by normal pieces. */
+ static const char *const special[] = {"<unk>", "<s>", "</s>"};
+ int expected_type = idx == 0 ? SP_PIECE_UNKNOWN : idx < 3 ? SP_PIECE_CONTROL : SP_PIECE_NORMAL;
+ if (!piece || type != expected_type || (idx < 3 && strcmp(piece, special[idx]) != 0))
goto fail;
tok->id_to_piece[idx] = piece;
tok->scores[idx] = score;
- tok->types[idx] = (uint8_t)type;
- if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len)
+ if (type == SP_PIECE_NORMAL && tok->piece_lens[idx] > tok->max_piece_len)
tok->max_piece_len = tok->piece_lens[idx];
return 0;
fail:
@@ -315,7 +308,7 @@ fail:
return -1;
}
-static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) {
+static int parse_trainer(const unsigned char *data, size_t len) {
const unsigned char *p = data, *end = data + len;
int model_type = SP_MODEL_UNIGRAM;
while (p < end) {
@@ -336,7 +329,6 @@ static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len)
model_type);
return -1;
}
- (void)tok;
return 0;
}
@@ -346,16 +338,12 @@ static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t le
int field = 0, wire = 0;
if (pb_read_key(&p, end, &field, &wire) != 0)
return -1;
+ /* NormalizerSpec fields 3..5 default to true: dummy prefix, whitespace
+ * merging, and escaping. XLM-R requires all three. */
if ((field == 3 || field == 4 || field == 5) && wire == 0) {
uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0)
+ if (pb_read_varint(&p, end, &v) != 0 || v != 1)
return -1;
- if (field == 3)
- tok->add_dummy_prefix = v != 0;
- else if (field == 4)
- tok->remove_extra_whitespaces = v != 0;
- else
- tok->escape_whitespaces = v != 0;
} else if (field == 2 && wire == 2) {
const unsigned char *blob = NULL;
size_t n = 0;
@@ -414,8 +402,7 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece));
tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens));
tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores));
- tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types));
- if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types)
+ if (!tok->id_to_piece || !tok->piece_lens || !tok->scores)
return -1;
const unsigned char *p = data, *end = data + len;
@@ -434,7 +421,7 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
} else if (wire == 2 && field == 2) {
const unsigned char *msg = NULL;
size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0)
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(msg, msg_len) != 0)
return -1;
saw_trainer = 1;
} else if (wire == 2 && field == 3) {
@@ -451,85 +438,6 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
return 0;
}
-static int raw_piece_id(const tok_spm_t *tok, const char *piece) {
- if (!tok || !piece)
- return -1;
- return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece));
-}
-
-static int raw_to_model_id(const tok_spm_t *tok, int raw_id) {
- if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
- return -1;
- if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
- if (raw_id == 1)
- return 0; /* <s> */
- if (raw_id == 2)
- return 2; /* </s> */
- if (raw_id == 0)
- return 3; /* <unk> */
- return raw_id + 1;
- }
- return raw_id;
-}
-
-static int file_contains_xlm_roberta(const char *model_dir, const char *name) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return 0;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return 0;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf) {
- fclose(f);
- return 0;
- }
- size_t got = fread(buf, 1, (size_t)sz, f);
- fclose(f);
- buf[got] = '\0';
- int yes = strstr(buf, "XLMRobertaTokenizer") != NULL ||
- strstr(buf, "XLMRobertaTokenizerFast") != NULL ||
- strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL;
- free(buf);
- return yes;
-}
-
-static int detect_xlm_roberta(const char *model_dir) {
- return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") ||
- file_contains_xlm_roberta(model_dir, "config.json");
-}
-
-static int find_model_file(const char *model_dir, char path[1024]) {
- static const char *names[] = {
- "sentencepiece.bpe.model",
- "tokenizer.model",
- "spiece.model",
- };
- for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) {
- int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]);
- if (n < 0 || n >= 1024)
- return -1;
- FILE *f = fopen(path, "rb");
- if (f) {
- fclose(f);
- return 0;
- }
- if (errno != ENOENT)
- return -1;
- }
- return -1;
-}
-
static int load_file(const char *path, unsigned char **out, size_t *out_len) {
FILE *f = fopen(path, "rb");
if (!f)
@@ -624,10 +532,7 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c
return -1;
size_t input_len = strlen(text);
const char space[] = "\xe2\x96\x81"; /* U+2581 */
- const char *space_s = tok->escape_whitespaces ? space : " ";
- size_t space_n = tok->escape_whitespaces ? 3 : 1;
int processing_non_ws = 0;
- int space_prepended = 0;
for (size_t off = 0; off < input_len;) {
norm_prefix_t r = normalize_prefix(tok, text, input_len, off);
@@ -636,22 +541,13 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c
if (c != ' ') {
if (!processing_non_ws) {
processing_non_ws = 1;
- if ((tok->add_dummy_prefix && !space_prepended) ||
- tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
- 0)
- return -1;
- space_prepended = 1;
- }
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space, 3) != 0)
+ return -1;
}
if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0)
return -1;
} else {
processing_non_ws = 0;
- if (!tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0)
- return -1;
- }
}
}
off += r.consumed ? r.consumed : 1;
@@ -663,8 +559,8 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c
static double compute_unknown_score(const tok_spm_t *tok) {
float min_score = FLT_MAX;
- for (int i = 0; i < tok->piece_count; i++) {
- if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
+ for (int i = 3; i < tok->piece_count; i++) {
+ if (tok->scores[i] < min_score)
min_score = tok->scores[i];
}
if (min_score == FLT_MAX)
@@ -680,8 +576,8 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
reserve_ids(ws, (int)n + 1) != 0)
return -1;
for (size_t i = 0; i <= n; i++)
- ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX};
- ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0};
+ ws->best[i] = (sp_best_t){TOK_XLMR_UNK_ID, -1, -DBL_MAX};
+ ws->best[0] = (sp_best_t){TOK_XLMR_UNK_ID, 0, 0.0};
double unk_score = tok->unknown_score;
for (size_t off = 0; off < n;) {
@@ -697,13 +593,13 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off,
(int)(end - off));
- if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
+ if (raw >= 0) {
if (end - off == cp_len)
single_cp_found = 1;
- double sc =
- cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]);
+ double sc = cur.score + tok->scores[raw];
+ /* Normal XLM-R token IDs are the SentencePiece IDs plus one. */
if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc};
+ ws->best[end] = (sp_best_t){raw + 1, (int)off, sc};
}
if (end == n)
break;
@@ -715,7 +611,7 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
size_t end = off + cp_len;
double sc = cur.score + unk_score;
if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc};
+ ws->best[end] = (sp_best_t){TOK_XLMR_UNK_ID, (int)off, sc};
}
off += cp_len;
}
@@ -726,7 +622,7 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
sp_best_t b = ws->best[off];
if (b.prev < 0 || b.prev >= (int)off)
return -1;
- int is_unk = b.id == tok->unk_id;
+ int is_unk = b.id == TOK_XLMR_UNK_ID;
if (!(prev_unk && is_unk)) {
if (out_n >= ws->ids_cap)
return -1;
@@ -747,8 +643,9 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
if (!model_dir)
return NULL;
char path[1024];
- if (find_model_file(model_dir, path) != 0) {
- fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir);
+ int length = snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir);
+ if (length < 0 || (size_t)length >= sizeof(path)) {
+ fprintf(stderr, "sentencepiece: model path is too long\n");
return NULL;
}
unsigned char *data = NULL;
@@ -763,11 +660,8 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
free(data);
return NULL;
}
- tok->add_dummy_prefix = 1;
- tok->remove_extra_whitespaces = 1;
- tok->escape_whitespaces = 1;
- tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY;
- if (parse_model(tok, data, len) != 0) {
+ if (parse_model(tok, data, len) != 0 || tok->piece_count < 4 ||
+ tok->max_piece_len <= 0 || !tok->xcda || tok->xcda_count == 0) {
fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
free(data);
tok_spm_free(tok);
@@ -782,7 +676,7 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
return NULL;
}
tok->piece_map_cap = cap;
- for (int i = 0; i < tok->piece_count; i++) {
+ for (int i = 3; i < tok->piece_count; i++) {
if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
tok->piece_lens[i], i) != 0) {
tok_spm_free(tok);
@@ -790,67 +684,10 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
}
}
- tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
- tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
- tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
- tok->pad_id =
- tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
- tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA
- ? tok->piece_count + 1
- : raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
- tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
tok->unknown_score = compute_unknown_score(tok);
- if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
- fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
- tok_spm_free(tok);
- return NULL;
- }
return tok;
}
-int tok_spm_token_id(const tok_spm_t *tok, const char *token) {
- if (!tok || !token)
- return -1;
- if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
- if (!strcmp(token, "<s>"))
- return 0;
- if (!strcmp(token, "<pad>"))
- return 1;
- if (!strcmp(token, "</s>"))
- return 2;
- if (!strcmp(token, "<unk>"))
- return 3;
- if (!strcmp(token, "<mask>"))
- return tok->piece_count + 1;
- }
- return raw_to_model_id(tok, raw_piece_id(tok, token));
-}
-
-static int tok_spm_encode_into(const tok_spm_t *tok,
- tok_spm_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return -1;
- if (normalize_into(tok, ws, text) != 0)
- return -1;
- int n = segment(tok, ws);
- if (n < 0)
- return -1;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (n == 0)
- return 0;
- if (!out_ids || out_cap < n)
- return -2;
- memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids));
- return 0;
-}
-
int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
tok_spm_workspace_t *ws,
const char *text,
@@ -859,9 +696,10 @@ int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
*out_n_tokens = -1;
if (!tok || !ws || !text)
return NULL;
- int n = 0;
- int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n);
- if (rc < 0 && rc != -2)
+ if (normalize_into(tok, ws, text) != 0)
+ return NULL;
+ int n = segment(tok, ws);
+ if (n < 0)
return NULL;
if (out_n_tokens)
*out_n_tokens = n;
@@ -897,7 +735,6 @@ void tok_spm_free(tok_spm_t *tok) {
free(tok->id_to_piece);
free(tok->piece_lens);
free(tok->scores);
- free(tok->types);
free(tok->piece_map);
free(tok->xcda);
free(tok->prefix_replacements);
diff --git a/liblangdet/sentencepiece.h b/liblangdet/sentencepiece.h
index 2230538..0d8cf44 100644
--- a/liblangdet/sentencepiece.h
+++ b/liblangdet/sentencepiece.h
@@ -1,10 +1,7 @@
/*
- * SentencePiece tokenizer for XLM-R style models.
- *
- * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model,
- * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R
- * and multilingual E5. It encodes text only; frontends add the model's special
- * token layout, e.g. <s> ... </s> for XLM-R.
+ * XLM-R SentencePiece Unigram tokenizer.
+ * Encodes text using the checkpoint's precompiled normalization table.
+ * The caller adds the leading <s> and trailing </s> tokens.
*/
#ifndef TOK_SPM_H
@@ -13,29 +10,19 @@
#include <stddef.h>
#include <stdint.h>
-typedef enum {
- TOK_SPM_IDENTITY = 0,
- TOK_SPM_XLM_ROBERTA = 1,
-} tok_spm_id_map_t;
+enum {
+ TOK_XLMR_BOS_ID = 0,
+ TOK_XLMR_EOS_ID = 2,
+ TOK_XLMR_UNK_ID = 3,
+};
typedef struct {
char **id_to_piece; /* [piece_count] raw SentencePiece ids */
uint16_t *piece_lens; /* byte length for id_to_piece entries */
float *scores; /* SentencePiece score per raw id */
- uint8_t *types; /* SentencePiece piece type per raw id */
int piece_count; /* raw SentencePiece vocabulary size */
- int vocab_size; /* model embedding vocab size after id mapping */
- int unk_id; /* model-space unknown-token id */
- int bos_id; /* model-space beginning-token id, or -1 */
- int eos_id; /* model-space end-token id, or -1 */
- int pad_id; /* model-space pad-token id, or -1 */
- int mask_id; /* model-space mask-token id, or -1 */
- int max_piece_len; /* max normal/user/unused piece byte length */
+ int max_piece_len; /* longest normal piece in bytes */
double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
- int add_dummy_prefix;
- int remove_extra_whitespaces;
- int escape_whitespaces;
- tok_spm_id_map_t id_map;
void *piece_map; /* piece bytes -> raw SentencePiece id */
int piece_map_cap;
@@ -48,14 +35,10 @@ typedef struct {
typedef struct tok_spm_workspace tok_spm_workspace_t;
-/* Load the first supported SentencePiece model found in model_dir. Returns NULL
- * on missing model file, unsupported model type, malformed protobuf, or OOM. */
+/* Load model_dir/sentencepiece.bpe.model with XLM-R vocabulary and normalization.
+ * Return NULL on missing or unsupported model, malformed protobuf, or OOM. */
tok_spm_t *tok_spm_load(const char *model_dir);
-/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
- * model-space id, or -1 when absent. */
-int tok_spm_token_id(const tok_spm_t *tok, const char *token);
-
tok_spm_workspace_t *tok_spm_workspace_new(void);
void tok_spm_workspace_free(tok_spm_workspace_t *ws);
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
index 96a5f42..8b4344c 100644
--- a/liblangdet/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -9,8 +9,6 @@
struct langdet_tok {
tok_spm_t *spm;
tok_spm_workspace_t *workspace;
- int cls_id;
- int sep_id;
};
static void set_err(char *err, size_t errlen, const char *msg) {
@@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen)
set_err(err, errlen, "out of memory");
goto fail;
}
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
return t;
fail:
@@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
return NULL;
}
memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
+ ids[0] = TOK_XLMR_BOS_ID;
+ ids[n + 1] = TOK_XLMR_EOS_ID;
if (n_out)
*n_out = n + 2;
return ids;
diff --git a/tools/test_safetensors.c b/tools/test_safetensors.c
new file mode 100644
index 0000000..5a00c54
--- /dev/null
+++ b/tools/test_safetensors.c
@@ -0,0 +1,74 @@
+#define _POSIX_C_SOURCE 200809L
+#include "safetensors.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define CHECK(condition) do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
+ exit(1); \
+ } \
+} while (0)
+
+static safetensors_file_t *read_header(const char *header) {
+ char path[] = "build/safetensors-test-XXXXXX";
+ int fd = mkstemp(path);
+ CHECK(fd >= 0);
+ FILE *file = fdopen(fd, "wb");
+ CHECK(file);
+ size_t length = strlen(header);
+ uint64_t padded = (length + 7) & ~(uint64_t)7;
+ const float values[] = {1.25f, -2.5f};
+ CHECK(fwrite(&padded, sizeof(padded), 1, file) == 1);
+ CHECK(fwrite(header, 1, length, file) == length);
+ for (size_t i = length; i < padded; i++)
+ CHECK(fputc(' ', file) != EOF);
+ CHECK(fwrite(values, sizeof(values), 1, file) == 1);
+ CHECK(fclose(file) == 0);
+ safetensors_file_t *sf = safetensors_open(path);
+ CHECK(unlink(path) == 0);
+ return sf;
+}
+
+int main(void) {
+ safetensors_file_t *sf = read_header(
+ "{\"__metadata__\":{\"format\":\"pt\"},"
+ "\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,8]}}");
+ CHECK(sf && sf->num_tensors == 1);
+ const safetensor_t *tensor = safetensors_find(sf, "weight");
+ CHECK(tensor && tensor->dtype == DTYPE_F32 && tensor->ndim == 1 && tensor->shape[0] == 2);
+ CHECK(tensor->data_offset == 0 && tensor->data_size == 8);
+ const float *values = safetensors_data(sf, tensor);
+ CHECK(values[0] == 1.25f && values[1] == -2.5f);
+ CHECK(!safetensors_find(sf, "missing"));
+ safetensors_close(sf);
+
+ sf = read_header("{\"position_ids\":{\"dtype\":\"I64\",\"shape\":[1,1],\"data_offsets\":[0,8]}}");
+ CHECK(sf && safetensors_find(sf, "position_ids")->dtype == DTYPE_UNKNOWN);
+ safetensors_close(sf);
+ safetensors_close(NULL);
+
+ const char *invalid[] = {
+ "{",
+ "[]",
+ "{\"weight\":null}",
+ "{\"weight\":{\"shape\":[2],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":2,\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[-1],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1.5],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[9223372036854775808],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[-1,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,0]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,9]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,18446744073709551616]}}",
+ };
+ for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++)
+ CHECK(!read_header(invalid[i]));
+ puts("PASS: single-file tensor loading, metadata, lookup, and malformed header rejection.");
+ return 0;
+}
diff --git a/tools/verify.py b/tools/verify.py
index 197f68f..d8bccf4 100644
--- a/tools/verify.py
+++ b/tools/verify.py
@@ -102,15 +102,33 @@ def check_failures(model_dir):
missing_tokenizer.mkdir()
shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json")
(missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors")
- for directory in (base / "absent", empty, invalid_config, missing_weights,
- corrupt_weights, missing_tokenizer):
+ directories = [base / "absent", empty, invalid_config, missing_weights,
+ corrupt_weights, missing_tokenizer]
+ for name, weights, tokenizer in (
+ ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"),
+ ("alternate-tokenizer", "model.safetensors", "tokenizer.model"),
+ ("alternate-spiece", "model.safetensors", "spiece.model"),
+ ("bert", "model.safetensors", "sentencepiece.bpe.model"),
+ ):
+ directory = base / name
+ directory.mkdir()
+ config = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
+ if name == "bert":
+ config["model_type"] = "bert"
+ config["architectures"] = ["BertForSequenceClassification"]
+ (directory / "config.json").write_text(json.dumps(config), encoding="utf-8")
+ (directory / weights).symlink_to(model_dir / "model.safetensors")
+ (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model")
+ directories.append(directory)
+ for directory in directories:
result = subprocess.run(
[str(CLI), "-d", str(directory), "hello"],
capture_output=True, timeout=30,
)
require(result.returncode == 1 and result.stderr and not result.stdout,
f"Failed model load did not return a clean error: {directory.name}")
- print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True)
+ print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.",
+ flush=True)
def check_reference(model_dir):
@@ -118,15 +136,14 @@ def check_reference(model_dir):
tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True)
require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer")
cases = list(CASES)
- # Adjacent sequence lengths exercise both sides of the 512-token limit.
+ # Boundary cases straddle the BLAS attention threshold and the token limit.
unit = "test "
require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1,
"Boundary fixture must encode to one token")
- cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513))
+ cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513))
texts = [text for _, text in cases]
native_ids = json_lines([TOKEN_IDS, model_dir, *texts])
require(len(native_ids) == len(cases), "Missing native tokenization results")
- reference_ids = []
for (name, text), ids in zip(cases, native_ids):
expected = tokenizer(text, truncation=False, verbose=False)["input_ids"]
if name.startswith("length-"):
@@ -134,7 +151,6 @@ def check_reference(model_dir):
if name == "long":
require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit")
require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}")
- reference_ids.append(expected)
print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True)
model = AutoModelForSequenceClassification.from_pretrained(