diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
| commit | 91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch) | |
| tree | b16f686279a1f1b0691a6c0d65a544084fcc1c1e /liblangdet/attn.c | |
| parent | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff) | |
| download | langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip | |
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'liblangdet/attn.c')
| -rw-r--r-- | liblangdet/attn.c | 246 |
1 files changed, 92 insertions, 154 deletions
diff --git a/liblangdet/attn.c b/liblangdet/attn.c index ac0f4d1..42d33de 100644 --- a/liblangdet/attn.c +++ b/liblangdet/attn.c @@ -12,112 +12,82 @@ # endif #endif -static inline float dot_f32_fast(const float *a, const float *b, int n) { - return dot_f32_impl(a, b, n); -} - -/* dst = dst * scale */ -static inline void vec_scale_inplace(float *dst, float scale, int n) { - vec_scale_inplace_impl(dst, scale, n); -} - -/* dst += alpha * src */ -static inline void vec_axpy_inplace(float *dst, const float *src, float alpha, int n) { - vec_axpy_inplace_impl(dst, src, alpha, n); -} - -/* dst = dst * correction + src */ -static inline void vec_scale_add(float *dst, const float *src, float correction, int n) { - vec_scale_add_impl(dst, src, correction, n); -} - -#define PACKED_ATTN_QUERY_TILE 32 -#define PACKED_ATTN_BLAS_MIN_SEQ 128 +#define ATTN_QUERY_TILE 32 +#define ATTN_BLAS_MIN_SEQ 128 -static void bidirectional_gqa_attention_packed_online_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; +static void attention_online_rows(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end) { + int hidden = n_heads * head_dim; for (int i = query_start; i < query_end; i++) { - const float *q_row = Q + (size_t)i * q_hidden + h * head_dim; - float *o_row = out + (size_t)i * q_hidden + h * head_dim; + const float *q_row = Q + (size_t)i * hidden + h * head_dim; + float *o_row = out + (size_t)i * hidden + h * head_dim; float max_score = -1e30f; float sum_exp = 0.0f; for (int d = 0; d < head_dim; d++) o_row[d] = 0.0f; - for (int j = start; j < end; j++) { - const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; - const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; + for (int j = 0; j < seq; j++) { + const float *k_row = K + (size_t)j * hidden + h * head_dim; + const float *v_row = V + (size_t)j * hidden + h * head_dim; - float score = dot_f32_fast(q_row, k_row, head_dim) * scale; + float score = dot_f32_impl(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); sum_exp = sum_exp * correction + 1.0f; - vec_scale_add(o_row, v_row, correction, head_dim); + vec_scale_add_impl(o_row, v_row, correction, head_dim); max_score = score; } else { float wt = expf(score - max_score); sum_exp += wt; - vec_axpy_inplace(o_row, v_row, wt, head_dim); + vec_axpy_inplace_impl(o_row, v_row, wt, head_dim); } } if (sum_exp > 0.0f) { float inv_sum = 1.0f / sum_exp; - vec_scale_inplace(o_row, inv_sum, head_dim); + vec_scale_inplace_impl(o_row, inv_sum, head_dim); } } } #ifdef USE_BLAS -static void bidirectional_gqa_attention_packed_blas_rows(float *out, - const float *Q, - const float *K, - const float *V, - int start, - int end, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - int h, - int query_start, - int query_end, - float *scores) { - int heads_per_kv = n_heads / n_kv_heads; - int q_hidden = n_heads * head_dim; - int kv_hidden = n_kv_heads * head_dim; - int kv_h = h / heads_per_kv; +static void attention_blas_rows(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + int h, + int query_start, + int query_end, + float *scores) { + int hidden = n_heads * head_dim; int rows = query_end - query_start; - int keys = end - start; - const float *q = Q + (size_t)query_start * q_hidden + h * head_dim; - const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim; - const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim; - float *o = out + (size_t)query_start * q_hidden + h * head_dim; + const float *q = Q + (size_t)query_start * hidden + h * head_dim; + const float *k = K + h * head_dim; + const float *v = V + h * head_dim; + float *o = out + (size_t)query_start * hidden + h * head_dim; - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k, - kv_hidden, 0.0f, scores, keys); - softmax(scores, rows, keys); - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys, - v, kv_hidden, 0.0f, o, q_hidden); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, seq, head_dim, scale, q, hidden, k, + hidden, 0.0f, scores, seq); + softmax(scores, rows, seq); + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, seq, 1.0f, scores, seq, + v, hidden, 0.0f, o, hidden); } #endif @@ -126,117 +96,85 @@ typedef struct { const float *Q; const float *K; const float *V; - const int *offsets; - int batch; - int n_heads, n_kv_heads; + int seq; + int n_heads; int head_dim; float scale; float *scores; - int max_seq; -} packed_gqa_attn_task_t; +} attention_task_t; -static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) { - packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg; +static void attention_worker(int tid, int n_threads, void *arg) { + attention_task_t *t = (attention_task_t *)arg; int task_id = 0; #ifdef USE_BLAS - float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL; + float *scores = t->scores ? t->scores + (size_t)tid * ATTN_QUERY_TILE * t->seq : NULL; #endif for (int h = 0; h < t->n_heads; h++) { - for (int b = 0; b < t->batch; b++) { - int start = t->offsets[b]; - int end = t->offsets[b + 1]; - for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) { - int q1 = q0 + PACKED_ATTN_QUERY_TILE; - if (q1 > end) - q1 = end; - if (task_id++ % n_threads != tid) - continue; + for (int q0 = 0; q0 < t->seq; q0 += ATTN_QUERY_TILE) { + int q1 = q0 + ATTN_QUERY_TILE; + if (q1 > t->seq) + q1 = t->seq; + if (task_id++ % n_threads != tid) + continue; #ifdef USE_BLAS - if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) { - bidirectional_gqa_attention_packed_blas_rows( - t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1, scores); - continue; - } -#endif - bidirectional_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, end, - t->n_heads, t->n_kv_heads, t->head_dim, - t->scale, h, q0, q1); + if (scores && t->seq >= ATTN_BLAS_MIN_SEQ) { + attention_blas_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads, + t->head_dim, t->scale, h, q0, q1, scores); + continue; } +#endif + attention_online_rows(t->out, t->Q, t->K, t->V, t->seq, t->n_heads, + t->head_dim, t->scale, h, q0, q1); } } } -size_t bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) { +size_t attention_scratch_bytes(int seq) { #ifdef USE_BLAS - if (!offsets || batch <= 0) + if (seq < ATTN_BLAS_MIN_SEQ) return 0; - - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - if (len > max_seq) - max_seq = len; - } - if (max_seq < PACKED_ATTN_BLAS_MIN_SEQ) + if ((size_t)seq > SIZE_MAX / (sizeof(float) * ATTN_QUERY_TILE * (size_t)tp_num_threads())) return 0; - if ((size_t)max_seq > - SIZE_MAX / (sizeof(float) * PACKED_ATTN_QUERY_TILE * (size_t)tp_num_threads())) - return 0; - return (size_t)tp_num_threads() * PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float); + return (size_t)tp_num_threads() * ATTN_QUERY_TILE * (size_t)seq * sizeof(float); #else - (void)offsets; - (void)batch; + (void)seq; return 0; #endif } -void bidirectional_gqa_attention_packed_with_scratch(float *out, - const float *Q, - const float *K, - const float *V, - const int *offsets, - int batch, - int n_heads, - int n_kv_heads, - int head_dim, - float scale, - float *scratch, - size_t scratch_bytes) { - long long qk_work = 0; - int max_seq = 0; - for (int b = 0; b < batch; b++) { - int len = offsets[b + 1] - offsets[b]; - qk_work += (long long)len * len * n_heads; - if (len > max_seq) - max_seq = len; - } - +void attention_with_scratch(float *out, + const float *Q, + const float *K, + const float *V, + int seq, + int n_heads, + int head_dim, + float scale, + float *scratch, + size_t scratch_bytes) { + long long qk_work = (long long)seq * seq * n_heads; float *scores = NULL; #ifdef USE_BLAS - size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch); + size_t required = attention_scratch_bytes(seq); if (required != 0 && scratch && scratch_bytes >= required) scores = scratch; #else (void)scratch; (void)scratch_bytes; #endif - packed_gqa_attn_task_t task = {.out = out, - .Q = Q, - .K = K, - .V = V, - .offsets = offsets, - .batch = batch, - .n_heads = n_heads, - .n_kv_heads = n_kv_heads, - .head_dim = head_dim, - .scale = scale, - .scores = scores, - .max_seq = max_seq}; + attention_task_t task = {.out = out, + .Q = Q, + .K = K, + .V = V, + .seq = seq, + .n_heads = n_heads, + .head_dim = head_dim, + .scale = scale, + .scores = scores}; if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) { - parallel_for(packed_gqa_attn_worker, &task); + parallel_for(attention_worker, &task); } else { - packed_gqa_attn_worker(0, 1, &task); + attention_worker(0, 1, &task); } } |
