aboutsummaryrefslogtreecommitdiff
path: root/liblangdet
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
commit91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch)
treeb16f686279a1f1b0691a6c0d65a544084fcc1c1e /liblangdet
parent6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff)
downloadlangdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz
langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'liblangdet')
-rw-r--r--liblangdet/attn.c246
-rw-r--r--liblangdet/forward.c8
-rw-r--r--liblangdet/kernels.h26
-rw-r--r--liblangdet/langdet.c4
-rw-r--r--liblangdet/model.c91
-rw-r--r--liblangdet/model_blas.c4
-rw-r--r--liblangdet/model_internal.h2
-rw-r--r--liblangdet/model_types.h2
-rw-r--r--liblangdet/safetensors.c466
-rw-r--r--liblangdet/safetensors.h27
-rw-r--r--liblangdet/sentencepiece.c229
-rw-r--r--liblangdet/sentencepiece.h39
-rw-r--r--liblangdet/tokenize.c12
13 files changed, 262 insertions, 894 deletions
diff --git a/liblangdet/attn.c b/liblangdet/attn.c
index ac0f4d1..42d33de 100644
--- a/liblangdet/attn.c
+++ b/liblangdet/attn.c
@@ -12,112 +12,82 @@
# endif
#endif
-static inline float dot_f32_fast(const float *a, const float *b, int n) {
- return dot_f32_impl(a, b, n);
-}
-
-/* dst = dst * scale */
-static inline void vec_scale_inplace(float *dst, float scale, int n) {
- vec_scale_inplace_impl(dst, scale, n);
-}
-
-/* dst += alpha * src */
-static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
- vec_axpy_inplace_impl(dst, src, alpha, n);
-}
-
-/* dst = dst * correction + src */
-static inline void vec_scale_add(float *dst, const float *src, float correction, int n) {
- vec_scale_add_impl(dst, src, correction, n);
-}
-
-#define PACKED_ATTN_QUERY_TILE 32
-#define PACKED_ATTN_BLAS_MIN_SEQ 128
+#define ATTN_QUERY_TILE 32
+#define ATTN_BLAS_MIN_SEQ 128
-static void bidirectional_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
+static void attention_online_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end) {
+ int hidden = n_heads * head_dim;
for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
+ const float *q_row = Q + (size_t)i * hidden + h * head_dim;
+ float *o_row = out + (size_t)i * hidden + h * head_dim;
float max_score = -1e30f;
float sum_exp = 0.0f;
for (int d = 0; d < head_dim; d++)
o_row[d] = 0.0f;
- for (int j = start; j < end; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
+ for (int j = 0; j < seq; j++) {
+ const float *k_row = K + (size_t)j * hidden + h * head_dim;
+ const float *v_row = V + (size_t)j * hidden + h * head_dim;
- float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
+ float score = dot_f32_impl(q_row, k_row, head_dim) * scale;
if (score > max_score) {
float correction = expf(max_score - score);
sum_exp = sum_exp * correction + 1.0f;
- vec_scale_add(o_row, v_row, correction, head_dim);
+ vec_scale_add_impl(o_row, v_row, correction, head_dim);
max_score = score;
} else {
float wt = expf(score - max_score);
sum_exp += wt;
- vec_axpy_inplace(o_row, v_row, wt, head_dim);
+ vec_axpy_inplace_impl(o_row, v_row, wt, head_dim);
}
}
if (sum_exp > 0.0f) {
float inv_sum = 1.0f / sum_exp;
- vec_scale_inplace(o_row, inv_sum, head_dim);
+ vec_scale_inplace_impl(o_row, inv_sum, head_dim);
}
}
}
#ifdef USE_BLAS
-static void bidirectional_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
+static void attention_blas_rows(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ int h,
+ int query_start,
+ int query_end,
+ float *scores) {
+ int hidden = n_heads * head_dim;
int rows = query_end - query_start;
- int keys = end - start;
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
+ const float *q = Q + (size_t)query_start * hidden + h * head_dim;
+ const float *k = K + h * head_dim;
+ const float *v = V + h * head_dim;
+ float *o = out + (size_t)query_start * hidden + h * head_dim;
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
- kv_hidden, 0.0f, scores, keys);
- softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, seq, head_dim, scale, q, hidden, k,
+ hidden, 0.0f, scores, seq);
+ softmax(scores, rows, seq);
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, seq, 1.0f, scores, seq,
+ v, hidden, 0.0f, o, hidden);
}
#endif
@@ -126,117 +96,85 @@ typedef struct {
const float *Q;
const float *K;
const float *V;
- const int *offsets;
- int batch;
- int n_heads, n_kv_heads;
+ int seq;
+ int n_heads;
int head_dim;
float scale;
float *scores;
- int max_seq;
-} packed_gqa_attn_task_t;
+} attention_task_t;
-static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
+static void attention_worker(int tid, int n_threads, void *arg) {
+ attention_task_t *t = (attention_task_t *)arg;
int task_id = 0;
#ifdef USE_BLAS
- float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
+ float *scores = t->scores ? t->scores + (size_t)tid * ATTN_QUERY_TILE * t->seq : NULL;
#endif
for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
+ for (int q0 = 0; q0 < t->seq; q0 += ATTN_QUERY_TILE) {
+ int q1 = q0 + ATTN_QUERY_TILE;
+ if (q1 > t->seq)
+ q1 = t->seq;
+ if (task_id++ % n_threads != tid)
+ continue;
#ifdef USE_BLAS
- if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
- bidirectional_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end,
- t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
+ if (scores && t->seq >= ATTN_BLAS_MIN_SEQ) {
+ attention_blas_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads,
+ t->head_dim, t->scale, h, q0, q1, scores);
+ continue;
}
+#endif
+ attention_online_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads,
+ t->head_dim, t->scale, h, q0, q1);
}
}
}
-size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
+size_t attention_scratch_bytes(int seq) {
#ifdef USE_BLAS
- if (!offsets || batch <= 0)
+ if (seq < ATTN_BLAS_MIN_SEQ)
return 0;
-
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- if (len > max_seq)
- max_seq = len;
- }
- if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ)
+ if ((size_t)seq > SIZE_MAX / (sizeof(float) * ATTN_QUERY_TILE * (size_t)tp_num_threads()))
return 0;
- if ((size_t)max_seq >
- SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads()))
- return 0;
- return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
+ return (size_t)tp_num_threads() * ATTN_QUERY_TILE * (size_t)seq * sizeof(float);
#else
- (void)offsets;
- (void)batch;
+ (void)seq;
return 0;
#endif
}
-void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * len * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
+void attention_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes) {
+ long long qk_work = (long long)seq * seq * n_heads;
float *scores = NULL;
#ifdef USE_BLAS
- size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ size_t required = attention_scratch_bytes(seq);
if (required != 0 && scratch && scratch_bytes >= required)
scores = scratch;
#else
(void)scratch;
(void)scratch_bytes;
#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
+ attention_task_t task = {.out = out,
+ .Q = Q,
+ .K = K,
+ .V = V,
+ .seq = seq,
+ .n_heads = n_heads,
+ .head_dim = head_dim,
+ .scale = scale,
+ .scores = scores};
if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
- parallel_for(packed_gqa_attn_worker, &task);
+ parallel_for(attention_worker, &task);
} else {
- packed_gqa_attn_worker(0, 1, &task);
+ attention_worker(0, 1, &task);
}
}
diff --git a/liblangdet/forward.c b/liblangdet/forward.c
index 14b1327..cede404 100644
--- a/liblangdet/forward.c
+++ b/liblangdet/forward.c
@@ -40,8 +40,7 @@ int langdet_model_encode_into(const langdet_model_t *model,
if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter ||
ensure_buffers(ws, cfg, n_tokens) != 0)
return -1;
- int offsets[2] = {0, n_tokens};
- ensure_attention_scores(ws, offsets, 1);
+ ensure_attention_scores(ws, n_tokens);
float *x = ws->x;
float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v;
@@ -73,9 +72,8 @@ int langdet_model_encode_into(const langdet_model_t *model,
add_bias_rows(k_buf, l->k_bias, n_tokens, hidden);
add_bias_rows(v_buf, l->v_bias, n_tokens, hidden);
- bidirectional_gqa_attention_packed_with_scratch(
- attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale,
- ws->attn_scores, ws->attn_scores_bytes);
+ attention_with_scratch(attn_out, q_buf, k_buf, v_buf, n_tokens, n_heads, head_dim, scale,
+ ws->attn_scores, ws->attn_scores_bytes);
linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden);
add_bias_rows(proj_out, l->o_bias, n_tokens, hidden);
diff --git a/liblangdet/kernels.h b/liblangdet/kernels.h
index 1025602..95f5df0 100644
--- a/liblangdet/kernels.h
+++ b/liblangdet/kernels.h
@@ -23,20 +23,18 @@ void layer_norm(float *out,
void softmax(float *x, int rows, int cols);
void gelu_inplace(float *x, int n);
-/* Attention is block-diagonal: each token attends only within its offsets span.
+/* Bidirectional attention over one unpadded sequence with equal Q/K/V head counts.
* Missing scratch uses the online-softmax path instead of tiled BLAS. */
-size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
-void bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
+size_t attention_scratch_bytes(int seq);
+void attention_with_scratch(float *out,
+ const float *Q,
+ const float *K,
+ const float *V,
+ int seq,
+ int n_heads,
+ int head_dim,
+ float scale,
+ float *scratch,
+ size_t scratch_bytes);
#endif /* KERNELS_H */
diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c
index 9410ba2..0fe5433 100644
--- a/liblangdet/langdet.c
+++ b/liblangdet/langdet.c
@@ -36,8 +36,8 @@ static int set_error(char *err, size_t errlen, const char *format, ...) {
static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape,
int ndim, char *err, size_t errlen) {
- safetensors_file_t *file = NULL;
- const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file);
+ const safetensors_file_t *file = d->model->safetensors;
+ const safetensor_t *tensor = safetensors_find(file, name);
if (!tensor) {
set_error(err, errlen, "Missing classifier tensor: %s", name);
return NULL;
diff --git a/liblangdet/model.c b/liblangdet/model.c
index 146bbe7..53f3692 100644
--- a/liblangdet/model.c
+++ b/liblangdet/model.c
@@ -46,10 +46,9 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static const float *load_weight(multi_safetensors_t *ms, const char *name,
+static const float *load_weight(const safetensors_file_t *sf, const char *name,
const int64_t *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = safetensors_find(sf, name);
if (!t) {
fprintf(stderr, "langdet: tensor not found: %s\n", name);
return NULL;
@@ -59,20 +58,20 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name,
return safetensors_data(sf, t);
}
-static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
+static int load_xlm_roberta_weights(langdet_model_t *model, const safetensors_file_t *sf) {
const langdet_config_t *cfg = &model->config;
langdet_weights_t *w = &model->weights;
char name[256];
int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
- w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight",
+ w->embed_tokens = load_weight(sf, "roberta.embeddings.word_embeddings.weight",
(int64_t[]){cfg->vocab_size, hidden}, 2);
- w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight",
+ w->position_embeddings = load_weight(sf, "roberta.embeddings.position_embeddings.weight",
(int64_t[]){cfg->max_position_embeddings, hidden}, 2);
- w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight",
+ w->token_type_embeddings = load_weight(sf, "roberta.embeddings.token_type_embeddings.weight",
(int64_t[]){cfg->type_vocab_size, hidden}, 2);
- w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
- w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
+ w->langdet_ln_w = load_weight(sf, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1);
+ w->langdet_ln_b = load_weight(sf, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1);
if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings ||
!w->langdet_ln_w || !w->langdet_ln_b)
return -1;
@@ -84,42 +83,42 @@ static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) {
for (int i = 0; i < cfg->n_layers; i++) {
langdet_layer_t *l = &w->layers[i];
-#define BERT_W(field, suffix, d0, d1) \
- do { \
- const int64_t e[2] = {(d0), (d1)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 2); \
- if (!l->field) \
- return -1; \
+#define LOAD_MATRIX(field, suffix, d0, d1) \
+ do { \
+ const int64_t e[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(sf, name, e, 2); \
+ if (!l->field) \
+ return -1; \
} while (0)
-#define BERT_V(field, suffix, d0) \
- do { \
- const int64_t e[1] = {(d0)}; \
- snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
- l->field = load_weight(ms, name, e, 1); \
- if (!l->field) \
- return -1; \
+#define LOAD_VECTOR(field, suffix, d0) \
+ do { \
+ const int64_t e[1] = {(d0)}; \
+ snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \
+ l->field = load_weight(sf, name, e, 1); \
+ if (!l->field) \
+ return -1; \
} while (0)
- BERT_W(wq, "attention.self.query.weight", hidden, hidden);
- BERT_V(q_bias, "attention.self.query.bias", hidden);
- BERT_W(wk, "attention.self.key.weight", hidden, hidden);
- BERT_V(k_bias, "attention.self.key.bias", hidden);
- BERT_W(wv, "attention.self.value.weight", hidden, hidden);
- BERT_V(v_bias, "attention.self.value.bias", hidden);
- BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
- BERT_V(o_bias, "attention.output.dense.bias", hidden);
- BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
- BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
- BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
- BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
- BERT_W(down_proj, "output.dense.weight", hidden, inter);
- BERT_V(ffn_out_bias, "output.dense.bias", hidden);
- BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
- BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
+ LOAD_MATRIX(wq, "attention.self.query.weight", hidden, hidden);
+ LOAD_VECTOR(q_bias, "attention.self.query.bias", hidden);
+ LOAD_MATRIX(wk, "attention.self.key.weight", hidden, hidden);
+ LOAD_VECTOR(k_bias, "attention.self.key.bias", hidden);
+ LOAD_MATRIX(wv, "attention.self.value.weight", hidden, hidden);
+ LOAD_VECTOR(v_bias, "attention.self.value.bias", hidden);
+ LOAD_MATRIX(wo, "attention.output.dense.weight", hidden, hidden);
+ LOAD_VECTOR(o_bias, "attention.output.dense.bias", hidden);
+ LOAD_VECTOR(input_norm, "attention.output.LayerNorm.weight", hidden);
+ LOAD_VECTOR(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
+ LOAD_MATRIX(up_proj, "intermediate.dense.weight", inter, hidden);
+ LOAD_VECTOR(ffn_inter_bias, "intermediate.dense.bias", inter);
+ LOAD_MATRIX(down_proj, "output.dense.weight", hidden, inter);
+ LOAD_VECTOR(ffn_out_bias, "output.dense.bias", hidden);
+ LOAD_VECTOR(post_attn_norm, "output.LayerNorm.weight", hidden);
+ LOAD_VECTOR(ffn_ln_bias, "output.LayerNorm.bias", hidden);
-#undef BERT_W
-#undef BERT_V
+#undef LOAD_MATRIX
+#undef LOAD_VECTOR
}
return 0;
}
@@ -132,12 +131,16 @@ langdet_model_t *langdet_model_load(const char *model_dir) {
return NULL;
if (langdet_config_parse(&model->config, model_dir) != 0)
goto fail;
- model->safetensors = multi_safetensors_open(model_dir);
+ char path[1024];
+ int length = snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
+ if (length < 0 || (size_t)length >= sizeof(path))
+ goto fail;
+ model->safetensors = safetensors_open(path);
if (!model->safetensors) {
fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir);
goto fail;
}
- if (load_bert_weights(model, model->safetensors) != 0)
+ if (load_xlm_roberta_weights(model, model->safetensors) != 0)
goto fail;
return model;
@@ -149,7 +152,7 @@ fail:
void langdet_model_free(langdet_model_t *model) {
if (!model)
return;
- multi_safetensors_close(model->safetensors);
+ safetensors_close(model->safetensors);
free(model->weights.layers);
free(model);
}
diff --git a/liblangdet/model_blas.c b/liblangdet/model_blas.c
index 41f9237..d26db32 100644
--- a/liblangdet/model_blas.c
+++ b/liblangdet/model_blas.c
@@ -55,8 +55,8 @@ int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq)
return 0;
}
-void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) {
- size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+void ensure_attention_scores(langdet_workspace_t *ws, int seq) {
+ size_t bytes = attention_scratch_bytes(seq);
if (bytes <= ws->attn_scores_bytes)
return;
diff --git a/liblangdet/model_internal.h b/liblangdet/model_internal.h
index 73a7851..ed7f237 100644
--- a/liblangdet/model_internal.h
+++ b/liblangdet/model_internal.h
@@ -12,6 +12,6 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
int ndim);
int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq);
-void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch);
+void ensure_attention_scores(langdet_workspace_t *ws, int seq);
#endif /* LANGDET_MODEL_INTERNAL_H */
diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h
index 31d101a..aa46d39 100644
--- a/liblangdet/model_types.h
+++ b/liblangdet/model_types.h
@@ -36,7 +36,7 @@ struct langdet_model {
langdet_config_t config;
langdet_weights_t weights;
/* All weight pointers borrow this mapping until model teardown. */
- multi_safetensors_t *safetensors;
+ safetensors_file_t *safetensors;
};
struct langdet_workspace {
diff --git a/liblangdet/safetensors.c b/liblangdet/safetensors.c
index cfba76e..546f51d 100644
--- a/liblangdet/safetensors.c
+++ b/liblangdet/safetensors.c
@@ -1,267 +1,68 @@
#include "safetensors.h"
-#include <dirent.h>
+#include <yyjson.h>
+
#include <fcntl.h>
-#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
-/* Minimal JSON parser for safetensors header */
-/* TODO: consider replacing with yyjson */
-
-static void skip_whitespace(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
- (*p)++;
-}
-
-static int parse_string(const char **p, char *out, size_t max_len) {
- skip_whitespace(p);
- if (**p != '"')
- return -1;
- (*p)++;
- size_t i = 0;
- while (**p && **p != '"' && i < max_len - 1) {
- if (**p == '\\') {
- (*p)++;
- if (**p == 'n')
- out[i++] = '\n';
- else if (**p == 't')
- out[i++] = '\t';
- else if (**p == '"')
- out[i++] = '"';
- else if (**p == '\\')
- out[i++] = '\\';
- else
- out[i++] = **p;
- } else {
- out[i++] = **p;
- }
- (*p)++;
- }
- out[i] = '\0';
- if (**p != '"')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int parse_int(const char **p, int64_t *out) {
- skip_whitespace(p);
- int64_t val = 0;
- int neg = 0;
- if (**p == '-') {
- neg = 1;
- (*p)++;
- }
- if (**p < '0' || **p > '9')
- return -1;
- while (**p >= '0' && **p <= '9') {
- val = val * 10 + (**p - '0');
- (*p)++;
- }
- *out = neg ? -val : val;
- return 0;
-}
-
-static safetensor_dtype_t parse_dtype(const char *s) {
- if (strcmp(s, "F32") == 0)
- return DTYPE_F32;
- if (strcmp(s, "F16") == 0)
- return DTYPE_F16;
- if (strcmp(s, "BF16") == 0)
- return DTYPE_BF16;
- if (strcmp(s, "I32") == 0)
- return DTYPE_I32;
- if (strcmp(s, "I64") == 0)
- return DTYPE_I64;
- if (strcmp(s, "BOOL") == 0)
- return DTYPE_BOOL;
- return DTYPE_UNKNOWN;
-}
-
-static int parse_tensor_entry(const char **p, safetensor_t *t) {
- skip_whitespace(p);
- if (**p != '{')
- return -1;
- (*p)++;
-
- t->dtype = DTYPE_UNKNOWN;
- t->ndim = 0;
- t->data_offset = 0;
- t->data_size = 0;
-
- while (**p && **p != '}') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
-
- char key[64];
- if (parse_string(p, key, sizeof(key)) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ':')
- return -1;
- (*p)++;
- skip_whitespace(p);
-
- if (strcmp(key, "dtype") == 0) {
- char dtype_str[32];
- if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0)
- return -1;
- t->dtype = parse_dtype(dtype_str);
- } else if (strcmp(key, "shape") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- t->ndim = 0;
- while (**p && **p != ']') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
- if (t->ndim >= 8)
- return -1;
- if (parse_int(p, &t->shape[t->ndim]) != 0)
- return -1;
- t->ndim++;
- }
- if (**p != ']')
- return -1;
- (*p)++;
- } else if (strcmp(key, "data_offsets") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t start_i;
- if (parse_int(p, &start_i) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ',')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t end_i;
- if (parse_int(p, &end_i) != 0)
- return -1;
- if (start_i < 0 || end_i < start_i)
- return -1;
- t->data_offset = (size_t)start_i;
- t->data_size = (size_t)(end_i - start_i);
- skip_whitespace(p);
- if (**p != ']')
- return -1;
- (*p)++;
- } else {
- /* Skip unknown value */
- if (**p == '"') {
- (*p)++;
- while (**p && **p != '"') {
- if (**p == '\\')
- (*p)++;
- if (**p)
- (*p)++;
- }
- if (**p == '"')
- (*p)++;
- } else if (**p == '[') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '[')
- depth++;
- else if (**p == ']')
- depth--;
- (*p)++;
- }
- } else if (**p == '{') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '{')
- depth++;
- else if (**p == '}')
- depth--;
- (*p)++;
- }
- } else {
- while (**p && **p != ',' && **p != '}')
- (*p)++;
- }
- }
- }
- if (**p != '}')
- return -1;
- (*p)++;
- return 0;
-}
-
static int parse_header(safetensors_file_t *sf) {
- const char *p = sf->header_json;
- skip_whitespace(&p);
- if (*p != '{')
+ yyjson_doc *doc = yyjson_read((char *)sf->data + 8, sf->header_size, 0);
+ if (!doc)
return -1;
- p++;
-
- sf->num_tensors = 0;
-
- while (*p && *p != '}') {
- skip_whitespace(&p);
- if (*p == ',') {
- p++;
- continue;
- }
- if (*p == '}')
- break;
+ yyjson_val *root = yyjson_doc_get_root(doc);
+ if (!yyjson_is_obj(root))
+ goto fail;
- char name[256];
- if (parse_string(&p, name, sizeof(name)) != 0)
- return -1;
- skip_whitespace(&p);
- if (*p != ':')
- return -1;
- p++;
-
- if (strcmp(name, "__metadata__") == 0) {
- skip_whitespace(&p);
- if (*p == '{') {
- int depth = 1;
- p++;
- while (*p && depth > 0) {
- if (*p == '{')
- depth++;
- else if (*p == '}')
- depth--;
- p++;
- }
- }
+ size_t index, count;
+ yyjson_val *key, *entry;
+ yyjson_obj_foreach(root, index, count, key, entry) {
+ if (yyjson_equals_str(key, "__metadata__"))
continue;
- }
+ if (!yyjson_is_obj(entry) || sf->num_tensors >= SAFETENSORS_MAX_TENSORS)
+ goto fail;
+ safetensor_t *tensor = &sf->tensors[sf->num_tensors];
+ size_t name_len = yyjson_get_len(key);
+ if (name_len >= sizeof(tensor->name))
+ goto fail;
+ memcpy(tensor->name, yyjson_get_str(key), name_len + 1);
- if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) {
- fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n",
- sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS);
- return -1;
+ yyjson_val *dtype = yyjson_obj_get(entry, "dtype");
+ yyjson_val *shape = yyjson_obj_get(entry, "shape");
+ yyjson_val *offsets = yyjson_obj_get(entry, "data_offsets");
+ if (!yyjson_is_str(dtype) || !yyjson_is_arr(shape) ||
+ yyjson_arr_size(shape) > sizeof(tensor->shape) / sizeof(tensor->shape[0]) ||
+ !yyjson_is_arr(offsets) || yyjson_arr_size(offsets) != 2)
+ goto fail;
+ tensor->dtype = yyjson_equals_str(dtype, "F32") ? DTYPE_F32 : DTYPE_UNKNOWN;
+ tensor->ndim = (int)yyjson_arr_size(shape);
+ for (int i = 0; i < tensor->ndim; i++) {
+ yyjson_val *dim = yyjson_arr_get(shape, (size_t)i);
+ if (!yyjson_is_uint(dim) || yyjson_get_uint(dim) > INT64_MAX)
+ goto fail;
+ tensor->shape[i] = (int64_t)yyjson_get_uint(dim);
}
-
- safetensor_t *t = &sf->tensors[sf->num_tensors];
- snprintf(t->name, sizeof(t->name), "%s", name);
- if (parse_tensor_entry(&p, t) != 0)
- return -1;
+ yyjson_val *start = yyjson_arr_get(offsets, 0);
+ yyjson_val *end = yyjson_arr_get(offsets, 1);
+ if (!yyjson_is_uint(start) || !yyjson_is_uint(end))
+ goto fail;
+ uint64_t first = yyjson_get_uint(start), last = yyjson_get_uint(end);
+ if (first > last || last > sf->file_size - 8 - sf->header_size)
+ goto fail;
+ tensor->data_offset = (size_t)first;
+ tensor->data_size = (size_t)(last - first);
sf->num_tensors++;
}
- if (*p != '}')
- return -1;
+ yyjson_doc_free(doc);
return 0;
-}
-/* Single file operations */
+fail:
+ yyjson_doc_free(doc);
+ return -1;
+}
safetensors_file_t *safetensors_open(const char *path) {
int fd = open(path, O_RDONLY);
@@ -269,17 +70,12 @@ safetensors_file_t *safetensors_open(const char *path) {
return NULL;
struct stat st;
- if (fstat(fd, &st) < 0) {
+ if (fstat(fd, &st) < 0 || st.st_size < 8) {
close(fd);
return NULL;
}
size_t file_size = (size_t)st.st_size;
- if (file_size < 8) {
- close(fd);
- return NULL;
- }
-
void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
if (data == MAP_FAILED)
@@ -298,24 +94,13 @@ safetensors_file_t *safetensors_open(const char *path) {
return NULL;
}
- sf->path = strdup(path);
sf->data = data;
sf->file_size = file_size;
sf->header_size = (size_t)header_size;
-
- sf->header_json = malloc(header_size + 1);
- if (!sf->header_json) {
- safetensors_close(sf);
- return NULL;
- }
- memcpy(sf->header_json, (char *)data + 8, header_size);
- sf->header_json[header_size] = '\0';
-
if (parse_header(sf) != 0) {
safetensors_close(sf);
return NULL;
}
-
return sf;
}
@@ -324,160 +109,17 @@ void safetensors_close(safetensors_file_t *sf) {
return;
if (sf->data)
munmap(sf->data, sf->file_size);
- free(sf->path);
- free(sf->header_json);
free(sf);
}
-const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
- return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
-}
-
-/* Multi-shard operations */
-
-static int cmp_shard_names(const void *a, const void *b) {
- const char *sa = (const char *)a;
- const char *sb = (const char *)b;
- return strcmp(sa, sb);
-}
-
-static int parse_shard_name(const char *name, int *idx, int *total) {
- int i = 0, n = 0, consumed = 0;
- if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2)
- return -1;
- if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n)
- return -1;
- *idx = i;
- *total = n;
- return 0;
-}
-
-multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
- multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
- if (!ms)
- return NULL;
-
- char path[4096];
-
- /* Try single file first */
- snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
- safetensors_file_t *sf = safetensors_open(path);
- if (sf) {
- ms->shards[0] = sf;
- ms->num_shards = 1;
- return ms;
- }
-
- /* Scan directory for shard files */
- DIR *dir = opendir(model_dir);
- if (!dir) {
- free(ms);
- return NULL;
- }
-
- struct dirent *entry;
- char shard_names[SAFETENSORS_MAX_SHARDS][256];
- int seen[SAFETENSORS_MAX_SHARDS] = {0};
- int n_shards = 0;
- int expected_total = 0;
-
- while ((entry = readdir(dir)) != NULL) {
- if (strncmp(entry->d_name, "model-", 6) == 0 &&
- strstr(entry->d_name, ".safetensors") != NULL) {
- int shard_idx = 0, shard_total = 0;
- if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
- fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (shard_total > SAFETENSORS_MAX_SHARDS) {
- fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total,
- SAFETENSORS_MAX_SHARDS);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (expected_total == 0) {
- expected_total = shard_total;
- } else if (expected_total != shard_total) {
- fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n",
- entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (seen[shard_idx - 1]) {
- fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx);
- closedir(dir);
- free(ms);
- return NULL;
- }
- seen[shard_idx - 1] = 1;
-
- snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name);
- n_shards++;
- }
- }
- closedir(dir);
-
- if (n_shards == 0) {
- fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir);
- free(ms);
- return NULL;
- }
- if (expected_total != n_shards) {
- fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n",
- expected_total, n_shards, model_dir);
- free(ms);
- return NULL;
- }
- for (int i = 0; i < expected_total; i++) {
- if (!seen[i]) {
- fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1,
- expected_total, model_dir);
- free(ms);
- return NULL;
- }
- }
-
- /* Sort shard names to ensure consistent ordering */
- qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names);
-
- /* Open each shard */
- for (int i = 0; i < n_shards; i++) {
- snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]);
- ms->shards[i] = safetensors_open(path);
- if (!ms->shards[i]) {
- fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path);
- multi_safetensors_close(ms);
- return NULL;
- }
- }
- ms->num_shards = n_shards;
- return ms;
-}
-
-void multi_safetensors_close(multi_safetensors_t *ms) {
- if (!ms)
- return;
- for (int i = 0; i < ms->num_shards; i++) {
- safetensors_close(ms->shards[i]);
+const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name) {
+ for (int i = 0; i < sf->num_tensors; i++) {
+ if (strcmp(sf->tensors[i].name, name) == 0)
+ return &sf->tensors[i];
}
- free(ms);
+ return NULL;
}
-const safetensor_t *
-multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) {
- for (int s = 0; s < ms->num_shards; s++) {
- safetensors_file_t *sf = ms->shards[s];
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0) {
- if (out_sf)
- *out_sf = sf;
- return &sf->tensors[i];
- }
- }
- }
- return NULL;
+const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
+ return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
}
diff --git a/liblangdet/safetensors.h b/liblangdet/safetensors.h
index dca68a7..99f1cfb 100644
--- a/liblangdet/safetensors.h
+++ b/liblangdet/safetensors.h
@@ -1,7 +1,3 @@
-/*
- * Safetensors file format reader with multi-shard support.
- */
-
#ifndef SAFETENSORS_H
#define SAFETENSORS_H
@@ -9,15 +5,10 @@
#include <stdint.h>
#define SAFETENSORS_MAX_TENSORS 1024
-#define SAFETENSORS_MAX_SHARDS 8
+/* Unused checkpoint entries may have non-F32 dtypes. */
typedef enum {
DTYPE_F32 = 0,
- DTYPE_F16 = 1,
- DTYPE_BF16 = 2,
- DTYPE_I32 = 3,
- DTYPE_I64 = 4,
- DTYPE_BOOL = 5,
DTYPE_UNKNOWN = -1
} safetensor_dtype_t;
@@ -31,32 +22,18 @@ typedef struct {
} safetensor_t;
typedef struct {
- char *path;
void *data;
size_t file_size;
size_t header_size;
- char *header_json;
int num_tensors;
safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
} safetensors_file_t;
-/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
-typedef struct {
- safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
- int num_shards;
-} multi_safetensors_t;
-
/* Open a single safetensors file (memory-mapped) */
safetensors_file_t *safetensors_open(const char *path);
void safetensors_close(safetensors_file_t *sf);
-/* Open model from directory (auto-detects single file or multi-shard) */
-multi_safetensors_t *multi_safetensors_open(const char *model_dir);
-void multi_safetensors_close(multi_safetensors_t *ms);
-
-/* Find a tensor by name across all shards */
-const safetensor_t *
-multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf);
+const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name);
/* Get raw pointer to tensor data (within mmap'd region) */
const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c
index 33efab8..e60e913 100644
--- a/liblangdet/sentencepiece.c
+++ b/liblangdet/sentencepiece.c
@@ -10,7 +10,6 @@
#include "sentencepiece.h"
-#include <errno.h>
#include <float.h>
#include <limits.h>
#include <stdio.h>
@@ -22,9 +21,6 @@ enum {
SP_PIECE_NORMAL = 1,
SP_PIECE_UNKNOWN = 2,
SP_PIECE_CONTROL = 3,
- SP_PIECE_USER_DEFINED = 4,
- SP_PIECE_UNUSED = 5,
- SP_PIECE_BYTE = 6,
};
typedef struct {
@@ -96,11 +92,6 @@ static int map_get(const sp_entry_t *map, int cap, const char *key, int len) {
return -1;
}
-static int is_tokenizable_type(int type) {
- return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED ||
- type == SP_PIECE_BYTE;
-}
-
static int reserve_chars(char **buf, size_t *cap, size_t need) {
if (need <= *cap)
return 0;
@@ -302,12 +293,14 @@ static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_
goto fail;
}
}
- if (!piece)
+ /* XLM-R stores <unk>, <s>, and </s> first, followed by normal pieces. */
+ static const char *const special[] = {"<unk>", "<s>", "</s>"};
+ int expected_type = idx == 0 ? SP_PIECE_UNKNOWN : idx < 3 ? SP_PIECE_CONTROL : SP_PIECE_NORMAL;
+ if (!piece || type != expected_type || (idx < 3 && strcmp(piece, special[idx]) != 0))
goto fail;
tok->id_to_piece[idx] = piece;
tok->scores[idx] = score;
- tok->types[idx] = (uint8_t)type;
- if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len)
+ if (type == SP_PIECE_NORMAL && tok->piece_lens[idx] > tok->max_piece_len)
tok->max_piece_len = tok->piece_lens[idx];
return 0;
fail:
@@ -315,7 +308,7 @@ fail:
return -1;
}
-static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) {
+static int parse_trainer(const unsigned char *data, size_t len) {
const unsigned char *p = data, *end = data + len;
int model_type = SP_MODEL_UNIGRAM;
while (p < end) {
@@ -336,7 +329,6 @@ static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len)
model_type);
return -1;
}
- (void)tok;
return 0;
}
@@ -346,16 +338,12 @@ static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t le
int field = 0, wire = 0;
if (pb_read_key(&p, end, &field, &wire) != 0)
return -1;
+ /* NormalizerSpec fields 3..5 default to true: dummy prefix, whitespace
+ * merging, and escaping. XLM-R requires all three. */
if ((field == 3 || field == 4 || field == 5) && wire == 0) {
uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0)
+ if (pb_read_varint(&p, end, &v) != 0 || v != 1)
return -1;
- if (field == 3)
- tok->add_dummy_prefix = v != 0;
- else if (field == 4)
- tok->remove_extra_whitespaces = v != 0;
- else
- tok->escape_whitespaces = v != 0;
} else if (field == 2 && wire == 2) {
const unsigned char *blob = NULL;
size_t n = 0;
@@ -414,8 +402,7 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece));
tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens));
tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores));
- tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types));
- if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types)
+ if (!tok->id_to_piece || !tok->piece_lens || !tok->scores)
return -1;
const unsigned char *p = data, *end = data + len;
@@ -434,7 +421,7 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
} else if (wire == 2 && field == 2) {
const unsigned char *msg = NULL;
size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0)
+ if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(msg, msg_len) != 0)
return -1;
saw_trainer = 1;
} else if (wire == 2 && field == 3) {
@@ -451,85 +438,6 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) {
return 0;
}
-static int raw_piece_id(const tok_spm_t *tok, const char *piece) {
- if (!tok || !piece)
- return -1;
- return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece));
-}
-
-static int raw_to_model_id(const tok_spm_t *tok, int raw_id) {
- if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
- return -1;
- if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
- if (raw_id == 1)
- return 0; /* <s> */
- if (raw_id == 2)
- return 2; /* </s> */
- if (raw_id == 0)
- return 3; /* <unk> */
- return raw_id + 1;
- }
- return raw_id;
-}
-
-static int file_contains_xlm_roberta(const char *model_dir, const char *name) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return 0;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return 0;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf) {
- fclose(f);
- return 0;
- }
- size_t got = fread(buf, 1, (size_t)sz, f);
- fclose(f);
- buf[got] = '\0';
- int yes = strstr(buf, "XLMRobertaTokenizer") != NULL ||
- strstr(buf, "XLMRobertaTokenizerFast") != NULL ||
- strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL;
- free(buf);
- return yes;
-}
-
-static int detect_xlm_roberta(const char *model_dir) {
- return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") ||
- file_contains_xlm_roberta(model_dir, "config.json");
-}
-
-static int find_model_file(const char *model_dir, char path[1024]) {
- static const char *names[] = {
- "sentencepiece.bpe.model",
- "tokenizer.model",
- "spiece.model",
- };
- for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) {
- int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]);
- if (n < 0 || n >= 1024)
- return -1;
- FILE *f = fopen(path, "rb");
- if (f) {
- fclose(f);
- return 0;
- }
- if (errno != ENOENT)
- return -1;
- }
- return -1;
-}
-
static int load_file(const char *path, unsigned char **out, size_t *out_len) {
FILE *f = fopen(path, "rb");
if (!f)
@@ -624,10 +532,7 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c
return -1;
size_t input_len = strlen(text);
const char space[] = "\xe2\x96\x81"; /* U+2581 */
- const char *space_s = tok->escape_whitespaces ? space : " ";
- size_t space_n = tok->escape_whitespaces ? 3 : 1;
int processing_non_ws = 0;
- int space_prepended = 0;
for (size_t off = 0; off < input_len;) {
norm_prefix_t r = normalize_prefix(tok, text, input_len, off);
@@ -636,22 +541,13 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c
if (c != ' ') {
if (!processing_non_ws) {
processing_non_ws = 1;
- if ((tok->add_dummy_prefix && !space_prepended) ||
- tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
- 0)
- return -1;
- space_prepended = 1;
- }
+ if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space, 3) != 0)
+ return -1;
}
if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0)
return -1;
} else {
processing_non_ws = 0;
- if (!tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0)
- return -1;
- }
}
}
off += r.consumed ? r.consumed : 1;
@@ -663,8 +559,8 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c
static double compute_unknown_score(const tok_spm_t *tok) {
float min_score = FLT_MAX;
- for (int i = 0; i < tok->piece_count; i++) {
- if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
+ for (int i = 3; i < tok->piece_count; i++) {
+ if (tok->scores[i] < min_score)
min_score = tok->scores[i];
}
if (min_score == FLT_MAX)
@@ -680,8 +576,8 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
reserve_ids(ws, (int)n + 1) != 0)
return -1;
for (size_t i = 0; i <= n; i++)
- ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX};
- ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0};
+ ws->best[i] = (sp_best_t){TOK_XLMR_UNK_ID, -1, -DBL_MAX};
+ ws->best[0] = (sp_best_t){TOK_XLMR_UNK_ID, 0, 0.0};
double unk_score = tok->unknown_score;
for (size_t off = 0; off < n;) {
@@ -697,13 +593,13 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off,
(int)(end - off));
- if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
+ if (raw >= 0) {
if (end - off == cp_len)
single_cp_found = 1;
- double sc =
- cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]);
+ double sc = cur.score + tok->scores[raw];
+ /* Normal XLM-R token IDs are the SentencePiece IDs plus one. */
if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc};
+ ws->best[end] = (sp_best_t){raw + 1, (int)off, sc};
}
if (end == n)
break;
@@ -715,7 +611,7 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
size_t end = off + cp_len;
double sc = cur.score + unk_score;
if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc};
+ ws->best[end] = (sp_best_t){TOK_XLMR_UNK_ID, (int)off, sc};
}
off += cp_len;
}
@@ -726,7 +622,7 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) {
sp_best_t b = ws->best[off];
if (b.prev < 0 || b.prev >= (int)off)
return -1;
- int is_unk = b.id == tok->unk_id;
+ int is_unk = b.id == TOK_XLMR_UNK_ID;
if (!(prev_unk && is_unk)) {
if (out_n >= ws->ids_cap)
return -1;
@@ -747,8 +643,9 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
if (!model_dir)
return NULL;
char path[1024];
- if (find_model_file(model_dir, path) != 0) {
- fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir);
+ int length = snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir);
+ if (length < 0 || (size_t)length >= sizeof(path)) {
+ fprintf(stderr, "sentencepiece: model path is too long\n");
return NULL;
}
unsigned char *data = NULL;
@@ -763,11 +660,8 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
free(data);
return NULL;
}
- tok->add_dummy_prefix = 1;
- tok->remove_extra_whitespaces = 1;
- tok->escape_whitespaces = 1;
- tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY;
- if (parse_model(tok, data, len) != 0) {
+ if (parse_model(tok, data, len) != 0 || tok->piece_count < 4 ||
+ tok->max_piece_len <= 0 || !tok->xcda || tok->xcda_count == 0) {
fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
free(data);
tok_spm_free(tok);
@@ -782,7 +676,7 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
return NULL;
}
tok->piece_map_cap = cap;
- for (int i = 0; i < tok->piece_count; i++) {
+ for (int i = 3; i < tok->piece_count; i++) {
if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
tok->piece_lens[i], i) != 0) {
tok_spm_free(tok);
@@ -790,67 +684,10 @@ tok_spm_t *tok_spm_load(const char *model_dir) {
}
}
- tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
- tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
- tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
- tok->pad_id =
- tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
- tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA
- ? tok->piece_count + 1
- : raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
- tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
tok->unknown_score = compute_unknown_score(tok);
- if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
- fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
- tok_spm_free(tok);
- return NULL;
- }
return tok;
}
-int tok_spm_token_id(const tok_spm_t *tok, const char *token) {
- if (!tok || !token)
- return -1;
- if (tok->id_map == TOK_SPM_XLM_ROBERTA) {
- if (!strcmp(token, "<s>"))
- return 0;
- if (!strcmp(token, "<pad>"))
- return 1;
- if (!strcmp(token, "</s>"))
- return 2;
- if (!strcmp(token, "<unk>"))
- return 3;
- if (!strcmp(token, "<mask>"))
- return tok->piece_count + 1;
- }
- return raw_to_model_id(tok, raw_piece_id(tok, token));
-}
-
-static int tok_spm_encode_into(const tok_spm_t *tok,
- tok_spm_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return -1;
- if (normalize_into(tok, ws, text) != 0)
- return -1;
- int n = segment(tok, ws);
- if (n < 0)
- return -1;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (n == 0)
- return 0;
- if (!out_ids || out_cap < n)
- return -2;
- memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids));
- return 0;
-}
-
int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
tok_spm_workspace_t *ws,
const char *text,
@@ -859,9 +696,10 @@ int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
*out_n_tokens = -1;
if (!tok || !ws || !text)
return NULL;
- int n = 0;
- int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n);
- if (rc < 0 && rc != -2)
+ if (normalize_into(tok, ws, text) != 0)
+ return NULL;
+ int n = segment(tok, ws);
+ if (n < 0)
return NULL;
if (out_n_tokens)
*out_n_tokens = n;
@@ -897,7 +735,6 @@ void tok_spm_free(tok_spm_t *tok) {
free(tok->id_to_piece);
free(tok->piece_lens);
free(tok->scores);
- free(tok->types);
free(tok->piece_map);
free(tok->xcda);
free(tok->prefix_replacements);
diff --git a/liblangdet/sentencepiece.h b/liblangdet/sentencepiece.h
index 2230538..0d8cf44 100644
--- a/liblangdet/sentencepiece.h
+++ b/liblangdet/sentencepiece.h
@@ -1,10 +1,7 @@
/*
- * SentencePiece tokenizer for XLM-R style models.
- *
- * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model,
- * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R
- * and multilingual E5. It encodes text only; frontends add the model's special
- * token layout, e.g. <s> ... </s> for XLM-R.
+ * XLM-R SentencePiece Unigram tokenizer.
+ * Encodes text using the checkpoint's precompiled normalization table.
+ * The caller adds the leading <s> and trailing </s> tokens.
*/
#ifndef TOK_SPM_H
@@ -13,29 +10,19 @@
#include <stddef.h>
#include <stdint.h>
-typedef enum {
- TOK_SPM_IDENTITY = 0,
- TOK_SPM_XLM_ROBERTA = 1,
-} tok_spm_id_map_t;
+enum {
+ TOK_XLMR_BOS_ID = 0,
+ TOK_XLMR_EOS_ID = 2,
+ TOK_XLMR_UNK_ID = 3,
+};
typedef struct {
char **id_to_piece; /* [piece_count] raw SentencePiece ids */
uint16_t *piece_lens; /* byte length for id_to_piece entries */
float *scores; /* SentencePiece score per raw id */
- uint8_t *types; /* SentencePiece piece type per raw id */
int piece_count; /* raw SentencePiece vocabulary size */
- int vocab_size; /* model embedding vocab size after id mapping */
- int unk_id; /* model-space unknown-token id */
- int bos_id; /* model-space beginning-token id, or -1 */
- int eos_id; /* model-space end-token id, or -1 */
- int pad_id; /* model-space pad-token id, or -1 */
- int mask_id; /* model-space mask-token id, or -1 */
- int max_piece_len; /* max normal/user/unused piece byte length */
+ int max_piece_len; /* longest normal piece in bytes */
double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
- int add_dummy_prefix;
- int remove_extra_whitespaces;
- int escape_whitespaces;
- tok_spm_id_map_t id_map;
void *piece_map; /* piece bytes -> raw SentencePiece id */
int piece_map_cap;
@@ -48,14 +35,10 @@ typedef struct {
typedef struct tok_spm_workspace tok_spm_workspace_t;
-/* Load the first supported SentencePiece model found in model_dir. Returns NULL
- * on missing model file, unsupported model type, malformed protobuf, or OOM. */
+/* Load model_dir/sentencepiece.bpe.model with XLM-R vocabulary and normalization.
+ * Return NULL on missing or unsupported model, malformed protobuf, or OOM. */
tok_spm_t *tok_spm_load(const char *model_dir);
-/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
- * model-space id, or -1 when absent. */
-int tok_spm_token_id(const tok_spm_t *tok, const char *token);
-
tok_spm_workspace_t *tok_spm_workspace_new(void);
void tok_spm_workspace_free(tok_spm_workspace_t *ws);
diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c
index 96a5f42..8b4344c 100644
--- a/liblangdet/tokenize.c
+++ b/liblangdet/tokenize.c
@@ -9,8 +9,6 @@
struct langdet_tok {
tok_spm_t *spm;
tok_spm_workspace_t *workspace;
- int cls_id;
- int sep_id;
};
static void set_err(char *err, size_t errlen, const char *msg) {
@@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen)
set_err(err, errlen, "out of memory");
goto fail;
}
- t->cls_id = tok_spm_token_id(t->spm, "<s>");
- t->sep_id = tok_spm_token_id(t->spm, "</s>");
- if (t->cls_id < 0 || t->sep_id < 0) {
- set_err(err, errlen, "SentencePiece vocab missing <s>/</s>");
- goto fail;
- }
return t;
fail:
@@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) {
return NULL;
}
memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = t->cls_id;
- ids[n + 1] = t->sep_id;
+ ids[0] = TOK_XLMR_BOS_ID;
+ ids[n + 1] = TOK_XLMR_EOS_ID;
if (n_out)
*n_out = n + 2;
return ids;