diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
| commit | 91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch) | |
| tree | b16f686279a1f1b0691a6c0d65a544084fcc1c1e /liblangdet | |
| parent | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff) | |
| download | langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip | |
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'liblangdet')
| -rw-r--r-- | liblangdet/attn.c | 246 | ||||
| -rw-r--r-- | liblangdet/forward.c | 8 | ||||
| -rw-r--r-- | liblangdet/kernels.h | 26 | ||||
| -rw-r--r-- | liblangdet/langdet.c | 4 | ||||
| -rw-r--r-- | liblangdet/model.c | 91 | ||||
| -rw-r--r-- | liblangdet/model_blas.c | 4 | ||||
| -rw-r--r-- | liblangdet/model_internal.h | 2 | ||||
| -rw-r--r-- | liblangdet/model_types.h | 2 | ||||
| -rw-r--r-- | liblangdet/safetensors.c | 466 | ||||
| -rw-r--r-- | liblangdet/safetensors.h | 27 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.c | 229 | ||||
| -rw-r--r-- | liblangdet/sentencepiece.h | 39 | ||||
| -rw-r--r-- | liblangdet/tokenize.c | 12 |
13 files changed, 262 insertions, 894 deletions
diff --git a/liblangdet/attn.c b/liblangdet/attn.c index ac0f4d1..42d33de 100644 --- a/liblangdet/attn.c +++ b/liblangdet/attn.c @@ -12,112 +12,82 @@ # endif #endif -static inline float dot_f32_fast(const float *a, const float *b, int n) { - return dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void vec_scale_inplace(float *dst, float scale, int n) { - vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { - vec_scale_add_impl(dst, src, correction, n); -} - -#define PACKED_ATTN_QUERY_TILE 32 -#define PACKED_ATTN_BLAS_MIN_SEQ 128 +#define ATTN_QUERY_TILE 32 +#define ATTN_BLAS_MIN_SEQ 128 -static void bidirectional_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; +static void attention_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int hidden = n_heads * head_dim; for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; + const float *q_row = Q + (size_t)i * hidden + h * head_dim; + float *o_row = out + (size_t)i * hidden + h * head_dim; float max_score = -1e30f; float sum_exp = 0.0f; for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + for (int j = 0; j < seq; j++) { + const float *k_row = K + (size_t)j * hidden + h * head_dim; + const float *v_row = V + (size_t)j * hidden + h * head_dim; - float score = dot_f32_fast(q_row, k_row, head_dim) * scale; + float score = dot_f32_impl(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); sum_exp = sum_exp * correction + 1.0f; - vec_scale_add(o_row, v_row, correction, head_dim); + vec_scale_add_impl(o_row, v_row, correction, head_dim); max_score = score; } else { float wt = expf(score - max_score); sum_exp += wt; - vec_axpy_inplace(o_row, v_row, wt, head_dim); + vec_axpy_inplace_impl(o_row, v_row, wt, head_dim); } } if (sum_exp > 0.0f) { float inv_sum = 1.0f / sum_exp; - vec_scale_inplace(o_row, inv_sum, head_dim); + vec_scale_inplace_impl(o_row, inv_sum, head_dim); } } } #ifdef USE_BLAS -static void bidirectional_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; +static void attention_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int hidden = n_heads * head_dim; int rows = query_end - query_start; - int keys = end - start; - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; + const float *q = Q + (size_t)query_start * hidden + h * head_dim; + const float *k = K + h * head_dim; + const float *v = V + h * head_dim; + float *o = out + (size_t)query_start * hidden + h * head_dim; - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, - kv_hidden, 0.0f, scores, keys); - softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, seq, head_dim, scale, q, hidden, k, + hidden, 0.0f, scores, seq); + softmax(scores, rows, seq); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, seq, 1.0f, scores, seq, + v, hidden, 0.0f, o, hidden); } #endif @@ -126,117 +96,85 @@ typedef struct { const float *Q; const float *K; const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; + int seq; + int n_heads; int head_dim; float scale; float *scores; - int max_seq; -} packed_gqa_attn_task_t; +} attention_task_t; -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; +static void attention_worker(int tid, int n_threads, void *arg) { + attention_task_t *t = (attention_task_t *)arg; int task_id = 0; #ifdef USE_BLAS - float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; + float *scores = t->scores ? t->scores + (size_t)tid * ATTN_QUERY_TILE * t->seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; + for (int q0 = 0; q0 < t->seq; q0 += ATTN_QUERY_TILE) { + int q1 = q0 + ATTN_QUERY_TILE; + if (q1 > t->seq) + q1 = t->seq; + if (task_id++ % n_threads != tid) + continue; #ifdef USE_BLAS - if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { - bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1, scores); - continue; - } -#endif - bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, - t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); + if (scores && t->seq >= ATTN_BLAS_MIN_SEQ) { + attention_blas_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads, + t->head_dim, t->scale, h, q0, q1, scores); + continue; } +#endif + attention_online_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads, + t->head_dim, t->scale, h, q0, q1); } } } -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +size_t attention_scratch_bytes(int seq) { #ifdef USE_BLAS - if (!offsets || batch <= 0) + if (seq < ATTN_BLAS_MIN_SEQ) return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) - max_seq = len; - } - if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) + if ((size_t)seq > SIZE_MAX / (sizeof(float) * ATTN_QUERY_TILE * (size_t)tp_num_threads())) return 0; - if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) - return 0; - return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); + return (size_t)tp_num_threads() * ATTN_QUERY_TILE * (size_t)seq * sizeof(float); #else - (void)offsets; - (void)batch; + (void)seq; return 0; #endif } -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) - max_seq = len; - } - +void attention_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = (long long)seq * seq * n_heads; float *scores = NULL; #ifdef USE_BLAS - size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t required = attention_scratch_bytes(seq); if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; #else (void)scratch; (void)scratch_bytes; #endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; + attention_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .seq = seq, + .n_heads = n_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores}; if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); + parallel_for(attention_worker, &task); } else { - packed_gqa_attn_worker(0, 1, &task); + attention_worker(0, 1, &task); } } diff --git a/liblangdet/forward.c b/liblangdet/forward.c index 14b1327..cede404 100644 --- a/liblangdet/forward.c +++ b/liblangdet/forward.c @@ -40,8 +40,7 @@ int langdet_model_encode_into(const langdet_model_t *model, if (n_tokens > INT_MAX / hidden || n_tokens > INT_MAX / inter || ensure_buffers(ws, cfg, n_tokens) != 0) return -1; - int offsets[2] = {0, n_tokens}; - ensure_attention_scores(ws, offsets, 1); + ensure_attention_scores(ws, n_tokens); float *x = ws->x; float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v; @@ -73,9 +72,8 @@ int langdet_model_encode_into(const langdet_model_t *model, add_bias_rows(k_buf, l->k_bias, n_tokens, hidden); add_bias_rows(v_buf, l->v_bias, n_tokens, hidden); - bidirectional_gqa_attention_packed_with_scratch( - attn_out, q_buf, k_buf, v_buf, offsets, 1, n_heads, n_heads, head_dim, scale, - ws->attn_scores, ws->attn_scores_bytes); + attention_with_scratch(attn_out, q_buf, k_buf, v_buf, n_tokens, n_heads, head_dim, scale, + ws->attn_scores, ws->attn_scores_bytes); linear_nobias(proj_out, attn_out, l->wo, n_tokens, hidden, hidden); add_bias_rows(proj_out, l->o_bias, n_tokens, hidden); diff --git a/liblangdet/kernels.h b/liblangdet/kernels.h index 1025602..95f5df0 100644 --- a/liblangdet/kernels.h +++ b/liblangdet/kernels.h @@ -23,20 +23,18 @@ void layer_norm(float *out, void softmax(float *x, int rows, int cols); void gelu_inplace(float *x, int n); -/* Attention is block-diagonal: each token attends only within its offsets span. +/* Bidirectional attention over one unpadded sequence with equal Q/K/V head counts. * Missing scratch uses the online-softmax path instead of tiled BLAS. */ -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch); -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes); +size_t attention_scratch_bytes(int seq); +void attention_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes); #endif /* KERNELS_H */ diff --git a/liblangdet/langdet.c b/liblangdet/langdet.c index 9410ba2..0fe5433 100644 --- a/liblangdet/langdet.c +++ b/liblangdet/langdet.c @@ -36,8 +36,8 @@ static int set_error(char *err, size_t errlen, const char *format, ...) { static const float *load_tensor(langdet_t *d, const char *name, const int64_t *shape, int ndim, char *err, size_t errlen) { - safetensors_file_t *file = NULL; - const safetensor_t *tensor = multi_safetensors_find(d->model->safetensors, name, &file); + const safetensors_file_t *file = d->model->safetensors; + const safetensor_t *tensor = safetensors_find(file, name); if (!tensor) { set_error(err, errlen, "Missing classifier tensor: %s", name); return NULL; diff --git a/liblangdet/model.c b/liblangdet/model.c index 146bbe7..53f3692 100644 --- a/liblangdet/model.c +++ b/liblangdet/model.c @@ -46,10 +46,9 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } -static const float *load_weight(multi_safetensors_t *ms, const char *name, +static const float *load_weight(const safetensors_file_t *sf, const char *name, const int64_t *shape, int ndim) { - safetensors_file_t *sf = NULL; - const safetensor_t *t = multi_safetensors_find(ms, name, &sf); + const safetensor_t *t = safetensors_find(sf, name); if (!t) { fprintf(stderr, "langdet: tensor not found: %s\n", name); return NULL; @@ -59,20 +58,20 @@ static const float *load_weight(multi_safetensors_t *ms, const char *name, return safetensors_data(sf, t); } -static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) { +static int load_xlm_roberta_weights(langdet_model_t *model, const safetensors_file_t *sf) { const langdet_config_t *cfg = &model->config; langdet_weights_t *w = &model->weights; char name[256]; int hidden = cfg->hidden_size, inter = cfg->intermediate_size; - w->embed_tokens = load_weight(ms, "roberta.embeddings.word_embeddings.weight", + w->embed_tokens = load_weight(sf, "roberta.embeddings.word_embeddings.weight", (int64_t[]){cfg->vocab_size, hidden}, 2); - w->position_embeddings = load_weight(ms, "roberta.embeddings.position_embeddings.weight", + w->position_embeddings = load_weight(sf, "roberta.embeddings.position_embeddings.weight", (int64_t[]){cfg->max_position_embeddings, hidden}, 2); - w->token_type_embeddings = load_weight(ms, "roberta.embeddings.token_type_embeddings.weight", + w->token_type_embeddings = load_weight(sf, "roberta.embeddings.token_type_embeddings.weight", (int64_t[]){cfg->type_vocab_size, hidden}, 2); - w->langdet_ln_w = load_weight(ms, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); - w->langdet_ln_b = load_weight(ms, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); + w->langdet_ln_w = load_weight(sf, "roberta.embeddings.LayerNorm.weight", (int64_t[]){hidden}, 1); + w->langdet_ln_b = load_weight(sf, "roberta.embeddings.LayerNorm.bias", (int64_t[]){hidden}, 1); if (!w->embed_tokens || !w->position_embeddings || !w->token_type_embeddings || !w->langdet_ln_w || !w->langdet_ln_b) return -1; @@ -84,42 +83,42 @@ static int load_bert_weights(langdet_model_t *model, multi_safetensors_t *ms) { for (int i = 0; i < cfg->n_layers; i++) { langdet_layer_t *l = &w->layers[i]; -#define BERT_W(field, suffix, d0, d1) \ - do { \ - const int64_t e[2] = {(d0), (d1)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 2); \ - if (!l->field) \ - return -1; \ +#define LOAD_MATRIX(field, suffix, d0, d1) \ + do { \ + const int64_t e[2] = {(d0), (d1)}; \ + snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ + l->field = load_weight(sf, name, e, 2); \ + if (!l->field) \ + return -1; \ } while (0) -#define BERT_V(field, suffix, d0) \ - do { \ - const int64_t e[1] = {(d0)}; \ - snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ - l->field = load_weight(ms, name, e, 1); \ - if (!l->field) \ - return -1; \ +#define LOAD_VECTOR(field, suffix, d0) \ + do { \ + const int64_t e[1] = {(d0)}; \ + snprintf(name, sizeof(name), "roberta.encoder.layer.%d." suffix, i); \ + l->field = load_weight(sf, name, e, 1); \ + if (!l->field) \ + return -1; \ } while (0) - BERT_W(wq, "attention.self.query.weight", hidden, hidden); - BERT_V(q_bias, "attention.self.query.bias", hidden); - BERT_W(wk, "attention.self.key.weight", hidden, hidden); - BERT_V(k_bias, "attention.self.key.bias", hidden); - BERT_W(wv, "attention.self.value.weight", hidden, hidden); - BERT_V(v_bias, "attention.self.value.bias", hidden); - BERT_W(wo, "attention.output.dense.weight", hidden, hidden); - BERT_V(o_bias, "attention.output.dense.bias", hidden); - BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden); - BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); - BERT_W(up_proj, "intermediate.dense.weight", inter, hidden); - BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter); - BERT_W(down_proj, "output.dense.weight", hidden, inter); - BERT_V(ffn_out_bias, "output.dense.bias", hidden); - BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden); - BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden); + LOAD_MATRIX(wq, "attention.self.query.weight", hidden, hidden); + LOAD_VECTOR(q_bias, "attention.self.query.bias", hidden); + LOAD_MATRIX(wk, "attention.self.key.weight", hidden, hidden); + LOAD_VECTOR(k_bias, "attention.self.key.bias", hidden); + LOAD_MATRIX(wv, "attention.self.value.weight", hidden, hidden); + LOAD_VECTOR(v_bias, "attention.self.value.bias", hidden); + LOAD_MATRIX(wo, "attention.output.dense.weight", hidden, hidden); + LOAD_VECTOR(o_bias, "attention.output.dense.bias", hidden); + LOAD_VECTOR(input_norm, "attention.output.LayerNorm.weight", hidden); + LOAD_VECTOR(attn_ln_bias, "attention.output.LayerNorm.bias", hidden); + LOAD_MATRIX(up_proj, "intermediate.dense.weight", inter, hidden); + LOAD_VECTOR(ffn_inter_bias, "intermediate.dense.bias", inter); + LOAD_MATRIX(down_proj, "output.dense.weight", hidden, inter); + LOAD_VECTOR(ffn_out_bias, "output.dense.bias", hidden); + LOAD_VECTOR(post_attn_norm, "output.LayerNorm.weight", hidden); + LOAD_VECTOR(ffn_ln_bias, "output.LayerNorm.bias", hidden); -#undef BERT_W -#undef BERT_V +#undef LOAD_MATRIX +#undef LOAD_VECTOR } return 0; } @@ -132,12 +131,16 @@ langdet_model_t *langdet_model_load(const char *model_dir) { return NULL; if (langdet_config_parse(&model->config, model_dir) != 0) goto fail; - model->safetensors = multi_safetensors_open(model_dir); + char path[1024]; + int length = snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); + if (length < 0 || (size_t)length >= sizeof(path)) + goto fail; + model->safetensors = safetensors_open(path); if (!model->safetensors) { fprintf(stderr, "langdet_model_load: failed to open safetensors in %s\n", model_dir); goto fail; } - if (load_bert_weights(model, model->safetensors) != 0) + if (load_xlm_roberta_weights(model, model->safetensors) != 0) goto fail; return model; @@ -149,7 +152,7 @@ fail: void langdet_model_free(langdet_model_t *model) { if (!model) return; - multi_safetensors_close(model->safetensors); + safetensors_close(model->safetensors); free(model->weights.layers); free(model); } diff --git a/liblangdet/model_blas.c b/liblangdet/model_blas.c index 41f9237..d26db32 100644 --- a/liblangdet/model_blas.c +++ b/liblangdet/model_blas.c @@ -55,8 +55,8 @@ int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *c, int seq) return 0; } -void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch) { - size_t bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); +void ensure_attention_scores(langdet_workspace_t *ws, int seq) { + size_t bytes = attention_scratch_bytes(seq); if (bytes <= ws->attn_scores_bytes) return; diff --git a/liblangdet/model_internal.h b/liblangdet/model_internal.h index 73a7851..ed7f237 100644 --- a/liblangdet/model_internal.h +++ b/liblangdet/model_internal.h @@ -12,6 +12,6 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, int ndim); int ensure_buffers(langdet_workspace_t *ws, const langdet_config_t *cfg, int seq); -void ensure_attention_scores(langdet_workspace_t *ws, const int *offsets, int batch); +void ensure_attention_scores(langdet_workspace_t *ws, int seq); #endif /* LANGDET_MODEL_INTERNAL_H */ diff --git a/liblangdet/model_types.h b/liblangdet/model_types.h index 31d101a..aa46d39 100644 --- a/liblangdet/model_types.h +++ b/liblangdet/model_types.h @@ -36,7 +36,7 @@ struct langdet_model { langdet_config_t config; langdet_weights_t weights; /* All weight pointers borrow this mapping until model teardown. */ - multi_safetensors_t *safetensors; + safetensors_file_t *safetensors; }; struct langdet_workspace { diff --git a/liblangdet/safetensors.c b/liblangdet/safetensors.c index cfba76e..546f51d 100644 --- a/liblangdet/safetensors.c +++ b/liblangdet/safetensors.c @@ -1,267 +1,68 @@ #include "safetensors.h" -#include <dirent.h> +#include <yyjson.h> + #include <fcntl.h> -#include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/mman.h> #include <sys/stat.h> #include <unistd.h> -/* Minimal JSON parser for safetensors header */ -/* TODO: consider replacing with yyjson */ - -static void skip_whitespace(const char **p) { - while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t') - (*p)++; -} - -static int parse_string(const char **p, char *out, size_t max_len) { - skip_whitespace(p); - if (**p != '"') - return -1; - (*p)++; - size_t i = 0; - while (**p && **p != '"' && i < max_len - 1) { - if (**p == '\\') { - (*p)++; - if (**p == 'n') - out[i++] = '\n'; - else if (**p == 't') - out[i++] = '\t'; - else if (**p == '"') - out[i++] = '"'; - else if (**p == '\\') - out[i++] = '\\'; - else - out[i++] = **p; - } else { - out[i++] = **p; - } - (*p)++; - } - out[i] = '\0'; - if (**p != '"') - return -1; - (*p)++; - return 0; -} - -static int parse_int(const char **p, int64_t *out) { - skip_whitespace(p); - int64_t val = 0; - int neg = 0; - if (**p == '-') { - neg = 1; - (*p)++; - } - if (**p < '0' || **p > '9') - return -1; - while (**p >= '0' && **p <= '9') { - val = val * 10 + (**p - '0'); - (*p)++; - } - *out = neg ? -val : val; - return 0; -} - -static safetensor_dtype_t parse_dtype(const char *s) { - if (strcmp(s, "F32") == 0) - return DTYPE_F32; - if (strcmp(s, "F16") == 0) - return DTYPE_F16; - if (strcmp(s, "BF16") == 0) - return DTYPE_BF16; - if (strcmp(s, "I32") == 0) - return DTYPE_I32; - if (strcmp(s, "I64") == 0) - return DTYPE_I64; - if (strcmp(s, "BOOL") == 0) - return DTYPE_BOOL; - return DTYPE_UNKNOWN; -} - -static int parse_tensor_entry(const char **p, safetensor_t *t) { - skip_whitespace(p); - if (**p != '{') - return -1; - (*p)++; - - t->dtype = DTYPE_UNKNOWN; - t->ndim = 0; - t->data_offset = 0; - t->data_size = 0; - - while (**p && **p != '}') { - skip_whitespace(p); - if (**p == ',') { - (*p)++; - continue; - } - - char key[64]; - if (parse_string(p, key, sizeof(key)) != 0) - return -1; - skip_whitespace(p); - if (**p != ':') - return -1; - (*p)++; - skip_whitespace(p); - - if (strcmp(key, "dtype") == 0) { - char dtype_str[32]; - if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0) - return -1; - t->dtype = parse_dtype(dtype_str); - } else if (strcmp(key, "shape") == 0) { - if (**p != '[') - return -1; - (*p)++; - t->ndim = 0; - while (**p && **p != ']') { - skip_whitespace(p); - if (**p == ',') { - (*p)++; - continue; - } - if (t->ndim >= 8) - return -1; - if (parse_int(p, &t->shape[t->ndim]) != 0) - return -1; - t->ndim++; - } - if (**p != ']') - return -1; - (*p)++; - } else if (strcmp(key, "data_offsets") == 0) { - if (**p != '[') - return -1; - (*p)++; - skip_whitespace(p); - int64_t start_i; - if (parse_int(p, &start_i) != 0) - return -1; - skip_whitespace(p); - if (**p != ',') - return -1; - (*p)++; - skip_whitespace(p); - int64_t end_i; - if (parse_int(p, &end_i) != 0) - return -1; - if (start_i < 0 || end_i < start_i) - return -1; - t->data_offset = (size_t)start_i; - t->data_size = (size_t)(end_i - start_i); - skip_whitespace(p); - if (**p != ']') - return -1; - (*p)++; - } else { - /* Skip unknown value */ - if (**p == '"') { - (*p)++; - while (**p && **p != '"') { - if (**p == '\\') - (*p)++; - if (**p) - (*p)++; - } - if (**p == '"') - (*p)++; - } else if (**p == '[') { - int depth = 1; - (*p)++; - while (**p && depth > 0) { - if (**p == '[') - depth++; - else if (**p == ']') - depth--; - (*p)++; - } - } else if (**p == '{') { - int depth = 1; - (*p)++; - while (**p && depth > 0) { - if (**p == '{') - depth++; - else if (**p == '}') - depth--; - (*p)++; - } - } else { - while (**p && **p != ',' && **p != '}') - (*p)++; - } - } - } - if (**p != '}') - return -1; - (*p)++; - return 0; -} - static int parse_header(safetensors_file_t *sf) { - const char *p = sf->header_json; - skip_whitespace(&p); - if (*p != '{') + yyjson_doc *doc = yyjson_read((char *)sf->data + 8, sf->header_size, 0); + if (!doc) return -1; - p++; - - sf->num_tensors = 0; - - while (*p && *p != '}') { - skip_whitespace(&p); - if (*p == ',') { - p++; - continue; - } - if (*p == '}') - break; + yyjson_val *root = yyjson_doc_get_root(doc); + if (!yyjson_is_obj(root)) + goto fail; - char name[256]; - if (parse_string(&p, name, sizeof(name)) != 0) - return -1; - skip_whitespace(&p); - if (*p != ':') - return -1; - p++; - - if (strcmp(name, "__metadata__") == 0) { - skip_whitespace(&p); - if (*p == '{') { - int depth = 1; - p++; - while (*p && depth > 0) { - if (*p == '{') - depth++; - else if (*p == '}') - depth--; - p++; - } - } + size_t index, count; + yyjson_val *key, *entry; + yyjson_obj_foreach(root, index, count, key, entry) { + if (yyjson_equals_str(key, "__metadata__")) continue; - } + if (!yyjson_is_obj(entry) || sf->num_tensors >= SAFETENSORS_MAX_TENSORS) + goto fail; + safetensor_t *tensor = &sf->tensors[sf->num_tensors]; + size_t name_len = yyjson_get_len(key); + if (name_len >= sizeof(tensor->name)) + goto fail; + memcpy(tensor->name, yyjson_get_str(key), name_len + 1); - if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) { - fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n", - sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS); - return -1; + yyjson_val *dtype = yyjson_obj_get(entry, "dtype"); + yyjson_val *shape = yyjson_obj_get(entry, "shape"); + yyjson_val *offsets = yyjson_obj_get(entry, "data_offsets"); + if (!yyjson_is_str(dtype) || !yyjson_is_arr(shape) || + yyjson_arr_size(shape) > sizeof(tensor->shape) / sizeof(tensor->shape[0]) || + !yyjson_is_arr(offsets) || yyjson_arr_size(offsets) != 2) + goto fail; + tensor->dtype = yyjson_equals_str(dtype, "F32") ? DTYPE_F32 : DTYPE_UNKNOWN; + tensor->ndim = (int)yyjson_arr_size(shape); + for (int i = 0; i < tensor->ndim; i++) { + yyjson_val *dim = yyjson_arr_get(shape, (size_t)i); + if (!yyjson_is_uint(dim) || yyjson_get_uint(dim) > INT64_MAX) + goto fail; + tensor->shape[i] = (int64_t)yyjson_get_uint(dim); } - - safetensor_t *t = &sf->tensors[sf->num_tensors]; - snprintf(t->name, sizeof(t->name), "%s", name); - if (parse_tensor_entry(&p, t) != 0) - return -1; + yyjson_val *start = yyjson_arr_get(offsets, 0); + yyjson_val *end = yyjson_arr_get(offsets, 1); + if (!yyjson_is_uint(start) || !yyjson_is_uint(end)) + goto fail; + uint64_t first = yyjson_get_uint(start), last = yyjson_get_uint(end); + if (first > last || last > sf->file_size - 8 - sf->header_size) + goto fail; + tensor->data_offset = (size_t)first; + tensor->data_size = (size_t)(last - first); sf->num_tensors++; } - if (*p != '}') - return -1; + yyjson_doc_free(doc); return 0; -} -/* Single file operations */ +fail: + yyjson_doc_free(doc); + return -1; +} safetensors_file_t *safetensors_open(const char *path) { int fd = open(path, O_RDONLY); @@ -269,17 +70,12 @@ safetensors_file_t *safetensors_open(const char *path) { return NULL; struct stat st; - if (fstat(fd, &st) < 0) { + if (fstat(fd, &st) < 0 || st.st_size < 8) { close(fd); return NULL; } size_t file_size = (size_t)st.st_size; - if (file_size < 8) { - close(fd); - return NULL; - } - void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); if (data == MAP_FAILED) @@ -298,24 +94,13 @@ safetensors_file_t *safetensors_open(const char *path) { return NULL; } - sf->path = strdup(path); sf->data = data; sf->file_size = file_size; sf->header_size = (size_t)header_size; - - sf->header_json = malloc(header_size + 1); - if (!sf->header_json) { - safetensors_close(sf); - return NULL; - } - memcpy(sf->header_json, (char *)data + 8, header_size); - sf->header_json[header_size] = '\0'; - if (parse_header(sf) != 0) { safetensors_close(sf); return NULL; } - return sf; } @@ -324,160 +109,17 @@ void safetensors_close(safetensors_file_t *sf) { return; if (sf->data) munmap(sf->data, sf->file_size); - free(sf->path); - free(sf->header_json); free(sf); } -const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { - return (const char *)sf->data + 8 + sf->header_size + t->data_offset; -} - -/* Multi-shard operations */ - -static int cmp_shard_names(const void *a, const void *b) { - const char *sa = (const char *)a; - const char *sb = (const char *)b; - return strcmp(sa, sb); -} - -static int parse_shard_name(const char *name, int *idx, int *total) { - int i = 0, n = 0, consumed = 0; - if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2) - return -1; - if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n) - return -1; - *idx = i; - *total = n; - return 0; -} - -multi_safetensors_t *multi_safetensors_open(const char *model_dir) { - multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t)); - if (!ms) - return NULL; - - char path[4096]; - - /* Try single file first */ - snprintf(path, sizeof(path), "%s/model.safetensors", model_dir); - safetensors_file_t *sf = safetensors_open(path); - if (sf) { - ms->shards[0] = sf; - ms->num_shards = 1; - return ms; - } - - /* Scan directory for shard files */ - DIR *dir = opendir(model_dir); - if (!dir) { - free(ms); - return NULL; - } - - struct dirent *entry; - char shard_names[SAFETENSORS_MAX_SHARDS][256]; - int seen[SAFETENSORS_MAX_SHARDS] = {0}; - int n_shards = 0; - int expected_total = 0; - - while ((entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "model-", 6) == 0 && - strstr(entry->d_name, ".safetensors") != NULL) { - int shard_idx = 0, shard_total = 0; - if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) { - fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (shard_total > SAFETENSORS_MAX_SHARDS) { - fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total, - SAFETENSORS_MAX_SHARDS); - closedir(dir); - free(ms); - return NULL; - } - if (expected_total == 0) { - expected_total = shard_total; - } else if (expected_total != shard_total) { - fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n", - entry->d_name); - closedir(dir); - free(ms); - return NULL; - } - if (seen[shard_idx - 1]) { - fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx); - closedir(dir); - free(ms); - return NULL; - } - seen[shard_idx - 1] = 1; - - snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name); - n_shards++; - } - } - closedir(dir); - - if (n_shards == 0) { - fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir); - free(ms); - return NULL; - } - if (expected_total != n_shards) { - fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n", - expected_total, n_shards, model_dir); - free(ms); - return NULL; - } - for (int i = 0; i < expected_total; i++) { - if (!seen[i]) { - fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1, - expected_total, model_dir); - free(ms); - return NULL; - } - } - - /* Sort shard names to ensure consistent ordering */ - qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names); - - /* Open each shard */ - for (int i = 0; i < n_shards; i++) { - snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]); - ms->shards[i] = safetensors_open(path); - if (!ms->shards[i]) { - fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path); - multi_safetensors_close(ms); - return NULL; - } - } - ms->num_shards = n_shards; - return ms; -} - -void multi_safetensors_close(multi_safetensors_t *ms) { - if (!ms) - return; - for (int i = 0; i < ms->num_shards; i++) { - safetensors_close(ms->shards[i]); +const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name) { + for (int i = 0; i < sf->num_tensors; i++) { + if (strcmp(sf->tensors[i].name, name) == 0) + return &sf->tensors[i]; } - free(ms); + return NULL; } -const safetensor_t * -multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf) { - for (int s = 0; s < ms->num_shards; s++) { - safetensors_file_t *sf = ms->shards[s]; - for (int i = 0; i < sf->num_tensors; i++) { - if (strcmp(sf->tensors[i].name, name) == 0) { - if (out_sf) - *out_sf = sf; - return &sf->tensors[i]; - } - } - } - return NULL; +const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) { + return (const char *)sf->data + 8 + sf->header_size + t->data_offset; } diff --git a/liblangdet/safetensors.h b/liblangdet/safetensors.h index dca68a7..99f1cfb 100644 --- a/liblangdet/safetensors.h +++ b/liblangdet/safetensors.h @@ -1,7 +1,3 @@ -/* - * Safetensors file format reader with multi-shard support. - */ - #ifndef SAFETENSORS_H #define SAFETENSORS_H @@ -9,15 +5,10 @@ #include <stdint.h> #define SAFETENSORS_MAX_TENSORS 1024 -#define SAFETENSORS_MAX_SHARDS 8 +/* Unused checkpoint entries may have non-F32 dtypes. */ typedef enum { DTYPE_F32 = 0, - DTYPE_F16 = 1, - DTYPE_BF16 = 2, - DTYPE_I32 = 3, - DTYPE_I64 = 4, - DTYPE_BOOL = 5, DTYPE_UNKNOWN = -1 } safetensor_dtype_t; @@ -31,32 +22,18 @@ typedef struct { } safetensor_t; typedef struct { - char *path; void *data; size_t file_size; size_t header_size; - char *header_json; int num_tensors; safetensor_t tensors[SAFETENSORS_MAX_TENSORS]; } safetensors_file_t; -/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */ -typedef struct { - safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS]; - int num_shards; -} multi_safetensors_t; - /* Open a single safetensors file (memory-mapped) */ safetensors_file_t *safetensors_open(const char *path); void safetensors_close(safetensors_file_t *sf); -/* Open model from directory (auto-detects single file or multi-shard) */ -multi_safetensors_t *multi_safetensors_open(const char *model_dir); -void multi_safetensors_close(multi_safetensors_t *ms); - -/* Find a tensor by name across all shards */ -const safetensor_t * -multi_safetensors_find(const multi_safetensors_t *ms, const char *name, safetensors_file_t **out_sf); +const safetensor_t *safetensors_find(const safetensors_file_t *sf, const char *name); /* Get raw pointer to tensor data (within mmap'd region) */ const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t); diff --git a/liblangdet/sentencepiece.c b/liblangdet/sentencepiece.c index 33efab8..e60e913 100644 --- a/liblangdet/sentencepiece.c +++ b/liblangdet/sentencepiece.c @@ -10,7 +10,6 @@ #include "sentencepiece.h" -#include <errno.h> #include <float.h> #include <limits.h> #include <stdio.h> @@ -22,9 +21,6 @@ enum { SP_PIECE_NORMAL = 1, SP_PIECE_UNKNOWN = 2, SP_PIECE_CONTROL = 3, - SP_PIECE_USER_DEFINED = 4, - SP_PIECE_UNUSED = 5, - SP_PIECE_BYTE = 6, }; typedef struct { @@ -96,11 +92,6 @@ static int map_get(const sp_entry_t *map, int cap, const char *key, int len) { return -1; } -static int is_tokenizable_type(int type) { - return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED || - type == SP_PIECE_BYTE; -} - static int reserve_chars(char **buf, size_t *cap, size_t need) { if (need <= *cap) return 0; @@ -302,12 +293,14 @@ static int parse_piece(tok_spm_t *tok, int idx, const unsigned char *data, size_ goto fail; } } - if (!piece) + /* XLM-R stores <unk>, <s>, and </s> first, followed by normal pieces. */ + static const char *const special[] = {"<unk>", "<s>", "</s>"}; + int expected_type = idx == 0 ? SP_PIECE_UNKNOWN : idx < 3 ? SP_PIECE_CONTROL : SP_PIECE_NORMAL; + if (!piece || type != expected_type || (idx < 3 && strcmp(piece, special[idx]) != 0)) goto fail; tok->id_to_piece[idx] = piece; tok->scores[idx] = score; - tok->types[idx] = (uint8_t)type; - if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len) + if (type == SP_PIECE_NORMAL && tok->piece_lens[idx] > tok->max_piece_len) tok->max_piece_len = tok->piece_lens[idx]; return 0; fail: @@ -315,7 +308,7 @@ fail: return -1; } -static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) { +static int parse_trainer(const unsigned char *data, size_t len) { const unsigned char *p = data, *end = data + len; int model_type = SP_MODEL_UNIGRAM; while (p < end) { @@ -336,7 +329,6 @@ static int parse_trainer(tok_spm_t *tok, const unsigned char *data, size_t len) model_type); return -1; } - (void)tok; return 0; } @@ -346,16 +338,12 @@ static int parse_normalizer(tok_spm_t *tok, const unsigned char *data, size_t le int field = 0, wire = 0; if (pb_read_key(&p, end, &field, &wire) != 0) return -1; + /* NormalizerSpec fields 3..5 default to true: dummy prefix, whitespace + * merging, and escaping. XLM-R requires all three. */ if ((field == 3 || field == 4 || field == 5) && wire == 0) { uint64_t v = 0; - if (pb_read_varint(&p, end, &v) != 0) + if (pb_read_varint(&p, end, &v) != 0 || v != 1) return -1; - if (field == 3) - tok->add_dummy_prefix = v != 0; - else if (field == 4) - tok->remove_extra_whitespaces = v != 0; - else - tok->escape_whitespaces = v != 0; } else if (field == 2 && wire == 2) { const unsigned char *blob = NULL; size_t n = 0; @@ -414,8 +402,7 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece)); tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens)); tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores)); - tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types)); - if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types) + if (!tok->id_to_piece || !tok->piece_lens || !tok->scores) return -1; const unsigned char *p = data, *end = data + len; @@ -434,7 +421,7 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { } else if (wire == 2 && field == 2) { const unsigned char *msg = NULL; size_t msg_len = 0; - if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0) + if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(msg, msg_len) != 0) return -1; saw_trainer = 1; } else if (wire == 2 && field == 3) { @@ -451,85 +438,6 @@ static int parse_model(tok_spm_t *tok, const unsigned char *data, size_t len) { return 0; } -static int raw_piece_id(const tok_spm_t *tok, const char *piece) { - if (!tok || !piece) - return -1; - return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece, (int)strlen(piece)); -} - -static int raw_to_model_id(const tok_spm_t *tok, int raw_id) { - if (!tok || raw_id < 0 || raw_id >= tok->piece_count) - return -1; - if (tok->id_map == TOK_SPM_XLM_ROBERTA) { - if (raw_id == 1) - return 0; /* <s> */ - if (raw_id == 2) - return 2; /* </s> */ - if (raw_id == 0) - return 3; /* <unk> */ - return raw_id + 1; - } - return raw_id; -} - -static int file_contains_xlm_roberta(const char *model_dir, const char *name) { - char path[1024]; - int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name); - if (n < 0 || (size_t)n >= sizeof(path)) - return 0; - FILE *f = fopen(path, "rb"); - if (!f) - return 0; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return 0; - } - long sz = ftell(f); - if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return 0; - } - char *buf = (char *)malloc((size_t)sz + 1); - if (!buf) { - fclose(f); - return 0; - } - size_t got = fread(buf, 1, (size_t)sz, f); - fclose(f); - buf[got] = '\0'; - int yes = strstr(buf, "XLMRobertaTokenizer") != NULL || - strstr(buf, "XLMRobertaTokenizerFast") != NULL || - strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL; - free(buf); - return yes; -} - -static int detect_xlm_roberta(const char *model_dir) { - return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") || - file_contains_xlm_roberta(model_dir, "config.json"); -} - -static int find_model_file(const char *model_dir, char path[1024]) { - static const char *names[] = { - "sentencepiece.bpe.model", - "tokenizer.model", - "spiece.model", - }; - for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) { - int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]); - if (n < 0 || n >= 1024) - return -1; - FILE *f = fopen(path, "rb"); - if (f) { - fclose(f); - return 0; - } - if (errno != ENOENT) - return -1; - } - return -1; -} - static int load_file(const char *path, unsigned char **out, size_t *out_len) { FILE *f = fopen(path, "rb"); if (!f) @@ -624,10 +532,7 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c return -1; size_t input_len = strlen(text); const char space[] = "\xe2\x96\x81"; /* U+2581 */ - const char *space_s = tok->escape_whitespaces ? space : " "; - size_t space_n = tok->escape_whitespaces ? 3 : 1; int processing_non_ws = 0; - int space_prepended = 0; for (size_t off = 0; off < input_len;) { norm_prefix_t r = normalize_prefix(tok, text, input_len, off); @@ -636,22 +541,13 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c if (c != ' ') { if (!processing_non_ws) { processing_non_ws = 1; - if ((tok->add_dummy_prefix && !space_prepended) || - tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != - 0) - return -1; - space_prepended = 1; - } + if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space, 3) != 0) + return -1; } if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0) return -1; } else { processing_non_ws = 0; - if (!tok->remove_extra_whitespaces) { - if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) != 0) - return -1; - } } } off += r.consumed ? r.consumed : 1; @@ -663,8 +559,8 @@ static int normalize_into(const tok_spm_t *tok, tok_spm_workspace_t *ws, const c static double compute_unknown_score(const tok_spm_t *tok) { float min_score = FLT_MAX; - for (int i = 0; i < tok->piece_count; i++) { - if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score) + for (int i = 3; i < tok->piece_count; i++) { + if (tok->scores[i] < min_score) min_score = tok->scores[i]; } if (min_score == FLT_MAX) @@ -680,8 +576,8 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { reserve_ids(ws, (int)n + 1) != 0) return -1; for (size_t i = 0; i <= n; i++) - ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX}; - ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0}; + ws->best[i] = (sp_best_t){TOK_XLMR_UNK_ID, -1, -DBL_MAX}; + ws->best[0] = (sp_best_t){TOK_XLMR_UNK_ID, 0, 0.0}; double unk_score = tok->unknown_score; for (size_t off = 0; off < n;) { @@ -697,13 +593,13 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) { int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, ws->norm + off, (int)(end - off)); - if (raw >= 0 && is_tokenizable_type(tok->types[raw])) { + if (raw >= 0) { if (end - off == cp_len) single_cp_found = 1; - double sc = - cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]); + double sc = cur.score + tok->scores[raw]; + /* Normal XLM-R token IDs are the SentencePiece IDs plus one. */ if (sc > ws->best[end].score) - ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc}; + ws->best[end] = (sp_best_t){raw + 1, (int)off, sc}; } if (end == n) break; @@ -715,7 +611,7 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { size_t end = off + cp_len; double sc = cur.score + unk_score; if (sc > ws->best[end].score) - ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc}; + ws->best[end] = (sp_best_t){TOK_XLMR_UNK_ID, (int)off, sc}; } off += cp_len; } @@ -726,7 +622,7 @@ static int segment(tok_spm_t const *tok, tok_spm_workspace_t *ws) { sp_best_t b = ws->best[off]; if (b.prev < 0 || b.prev >= (int)off) return -1; - int is_unk = b.id == tok->unk_id; + int is_unk = b.id == TOK_XLMR_UNK_ID; if (!(prev_unk && is_unk)) { if (out_n >= ws->ids_cap) return -1; @@ -747,8 +643,9 @@ tok_spm_t *tok_spm_load(const char *model_dir) { if (!model_dir) return NULL; char path[1024]; - if (find_model_file(model_dir, path) != 0) { - fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir); + int length = snprintf(path, sizeof(path), "%s/sentencepiece.bpe.model", model_dir); + if (length < 0 || (size_t)length >= sizeof(path)) { + fprintf(stderr, "sentencepiece: model path is too long\n"); return NULL; } unsigned char *data = NULL; @@ -763,11 +660,8 @@ tok_spm_t *tok_spm_load(const char *model_dir) { free(data); return NULL; } - tok->add_dummy_prefix = 1; - tok->remove_extra_whitespaces = 1; - tok->escape_whitespaces = 1; - tok->id_map = detect_xlm_roberta(model_dir) ? TOK_SPM_XLM_ROBERTA : TOK_SPM_IDENTITY; - if (parse_model(tok, data, len) != 0) { + if (parse_model(tok, data, len) != 0 || tok->piece_count < 4 || + tok->max_piece_len <= 0 || !tok->xcda || tok->xcda_count == 0) { fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path); free(data); tok_spm_free(tok); @@ -782,7 +676,7 @@ tok_spm_t *tok_spm_load(const char *model_dir) { return NULL; } tok->piece_map_cap = cap; - for (int i = 0; i < tok->piece_count; i++) { + for (int i = 3; i < tok->piece_count; i++) { if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i], tok->piece_lens[i], i) != 0) { tok_spm_free(tok); @@ -790,67 +684,10 @@ tok_spm_t *tok_spm_load(const char *model_dir) { } } - tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>")); - tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>")); - tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>")); - tok->pad_id = - tok->id_map == TOK_SPM_XLM_ROBERTA ? 1 : raw_to_model_id(tok, raw_piece_id(tok, "<pad>")); - tok->mask_id = tok->id_map == TOK_SPM_XLM_ROBERTA - ? tok->piece_count + 1 - : raw_to_model_id(tok, raw_piece_id(tok, "<mask>")); - tok->vocab_size = tok->id_map == TOK_SPM_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count; tok->unknown_score = compute_unknown_score(tok); - if (tok->unk_id < 0 || tok->max_piece_len <= 0) { - fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path); - tok_spm_free(tok); - return NULL; - } return tok; } -int tok_spm_token_id(const tok_spm_t *tok, const char *token) { - if (!tok || !token) - return -1; - if (tok->id_map == TOK_SPM_XLM_ROBERTA) { - if (!strcmp(token, "<s>")) - return 0; - if (!strcmp(token, "<pad>")) - return 1; - if (!strcmp(token, "</s>")) - return 2; - if (!strcmp(token, "<unk>")) - return 3; - if (!strcmp(token, "<mask>")) - return tok->piece_count + 1; - } - return raw_to_model_id(tok, raw_piece_id(tok, token)); -} - -static int tok_spm_encode_into(const tok_spm_t *tok, - tok_spm_workspace_t *ws, - const char *text, - int *out_ids, - int out_cap, - int *out_n_tokens) { - if (out_n_tokens) - *out_n_tokens = 0; - if (!tok || !ws || !text) - return -1; - if (normalize_into(tok, ws, text) != 0) - return -1; - int n = segment(tok, ws); - if (n < 0) - return -1; - if (out_n_tokens) - *out_n_tokens = n; - if (n == 0) - return 0; - if (!out_ids || out_cap < n) - return -2; - memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids)); - return 0; -} - int *tok_spm_encode_with_workspace(const tok_spm_t *tok, tok_spm_workspace_t *ws, const char *text, @@ -859,9 +696,10 @@ int *tok_spm_encode_with_workspace(const tok_spm_t *tok, *out_n_tokens = -1; if (!tok || !ws || !text) return NULL; - int n = 0; - int rc = tok_spm_encode_into(tok, ws, text, NULL, 0, &n); - if (rc < 0 && rc != -2) + if (normalize_into(tok, ws, text) != 0) + return NULL; + int n = segment(tok, ws); + if (n < 0) return NULL; if (out_n_tokens) *out_n_tokens = n; @@ -897,7 +735,6 @@ void tok_spm_free(tok_spm_t *tok) { free(tok->id_to_piece); free(tok->piece_lens); free(tok->scores); - free(tok->types); free(tok->piece_map); free(tok->xcda); free(tok->prefix_replacements); diff --git a/liblangdet/sentencepiece.h b/liblangdet/sentencepiece.h index 2230538..0d8cf44 100644 --- a/liblangdet/sentencepiece.h +++ b/liblangdet/sentencepiece.h @@ -1,10 +1,7 @@ /* - * SentencePiece tokenizer for XLM-R style models. - * - * Loads a SentencePiece ModelProto (sentencepiece.bpe.model, tokenizer.model, - * or spiece.model) and implements the Unigram Viterbi encoder used by XLM-R - * and multilingual E5. It encodes text only; frontends add the model's special - * token layout, e.g. <s> ... </s> for XLM-R. + * XLM-R SentencePiece Unigram tokenizer. + * Encodes text using the checkpoint's precompiled normalization table. + * The caller adds the leading <s> and trailing </s> tokens. */ #ifndef TOK_SPM_H @@ -13,29 +10,19 @@ #include <stddef.h> #include <stdint.h> -typedef enum { - TOK_SPM_IDENTITY = 0, - TOK_SPM_XLM_ROBERTA = 1, -} tok_spm_id_map_t; +enum { + TOK_XLMR_BOS_ID = 0, + TOK_XLMR_EOS_ID = 2, + TOK_XLMR_UNK_ID = 3, +}; typedef struct { char **id_to_piece; /* [piece_count] raw SentencePiece ids */ uint16_t *piece_lens; /* byte length for id_to_piece entries */ float *scores; /* SentencePiece score per raw id */ - uint8_t *types; /* SentencePiece piece type per raw id */ int piece_count; /* raw SentencePiece vocabulary size */ - int vocab_size; /* model embedding vocab size after id mapping */ - int unk_id; /* model-space unknown-token id */ - int bos_id; /* model-space beginning-token id, or -1 */ - int eos_id; /* model-space end-token id, or -1 */ - int pad_id; /* model-space pad-token id, or -1 */ - int mask_id; /* model-space mask-token id, or -1 */ - int max_piece_len; /* max normal/user/unused piece byte length */ + int max_piece_len; /* longest normal piece in bytes */ double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */ - int add_dummy_prefix; - int remove_extra_whitespaces; - int escape_whitespaces; - tok_spm_id_map_t id_map; void *piece_map; /* piece bytes -> raw SentencePiece id */ int piece_map_cap; @@ -48,14 +35,10 @@ typedef struct { typedef struct tok_spm_workspace tok_spm_workspace_t; -/* Load the first supported SentencePiece model found in model_dir. Returns NULL - * on missing model file, unsupported model type, malformed protobuf, or OOM. */ +/* Load model_dir/sentencepiece.bpe.model with XLM-R vocabulary and normalization. + * Return NULL on missing or unsupported model, malformed protobuf, or OOM. */ tok_spm_t *tok_spm_load(const char *model_dir); -/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a - * model-space id, or -1 when absent. */ -int tok_spm_token_id(const tok_spm_t *tok, const char *token); - tok_spm_workspace_t *tok_spm_workspace_new(void); void tok_spm_workspace_free(tok_spm_workspace_t *ws); diff --git a/liblangdet/tokenize.c b/liblangdet/tokenize.c index 96a5f42..8b4344c 100644 --- a/liblangdet/tokenize.c +++ b/liblangdet/tokenize.c @@ -9,8 +9,6 @@ struct langdet_tok { tok_spm_t *spm; tok_spm_workspace_t *workspace; - int cls_id; - int sep_id; }; static void set_err(char *err, size_t errlen, const char *msg) { @@ -38,12 +36,6 @@ langdet_tok_t *langdet_tok_open(const char *model_dir, char *err, size_t errlen) set_err(err, errlen, "out of memory"); goto fail; } - t->cls_id = tok_spm_token_id(t->spm, "<s>"); - t->sep_id = tok_spm_token_id(t->spm, "</s>"); - if (t->cls_id < 0 || t->sep_id < 0) { - set_err(err, errlen, "SentencePiece vocab missing <s>/</s>"); - goto fail; - } return t; fail: @@ -77,8 +69,8 @@ int *langdet_tokenize(langdet_tok_t *t, const char *text, int *n_out) { return NULL; } memmove(ids + 1, ids, (size_t)n * sizeof(*ids)); - ids[0] = t->cls_id; - ids[n + 1] = t->sep_id; + ids[0] = TOK_XLMR_BOS_ID; + ids[n + 1] = TOK_XLMR_EOS_ID; if (n_out) *n_out = n + 2; return ids; |
