aboutsummaryrefslogtreecommitdiff
path: root/libffwd/attn.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 13:05:44 +0200
commitcf7d8169ec12cb3adf90d4ff78b3257e74985302 (patch)
treef666589acfd68e76bbde4e74eead44ae7819d0bb /libffwd/attn.c
parent91871be6ece9317c5d49353f6474583e7e28168d (diff)
downloadlangdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.tar.gz
langdet-cf7d8169ec12cb3adf90d4ff78b3257e74985302.zip
Remove unused model families and inference paths
Diffstat (limited to 'libffwd/attn.c')
-rw-r--r--libffwd/attn.c216
1 files changed, 0 insertions, 216 deletions
diff --git a/libffwd/attn.c b/libffwd/attn.c
index 57c665a..ac0f4d1 100644
--- a/libffwd/attn.c
+++ b/libffwd/attn.c
@@ -3,8 +3,6 @@
#include "threadpool.h"
#include <math.h>
-#include <stdlib.h>
-#include <string.h>
#ifdef USE_BLAS
# ifdef __APPLE__
@@ -14,17 +12,6 @@
# endif
#endif
-#ifndef M_PI
-# define M_PI 3.14159265358979323846
-#endif
-
-/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE)
- * GQA, each with an online-softmax and a BLAS-GEMM path. */
-
-/* Attention operations */
-
-float dot_f32(const float *a, const float *b, int n) { return dot_f32_impl(a, b, n); }
-
static inline float dot_f32_fast(const float *a, const float *b, int n) {
return dot_f32_impl(a, b, n);
}
@@ -253,206 +240,3 @@ void bidirectional_gqa_attention_packed_with_scratch(float *out,
packed_gqa_attn_worker(0, 1, &task);
}
}
-
-void bidirectional_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale) {
- size_t scratch_bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- bidirectional_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
- head_dim, scale, scores, scratch_bytes);
- free(scores);
-}
-
-static void causal_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
-
- for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
-
- float max_score = -1e30f;
- float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++)
- o_row[d] = 0.0f;
-
- for (int j = start; j <= i; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
- float score = dot_f32_fast(q_row, k_row, head_dim) * scale;
-
- if (score > max_score) {
- float correction = expf(max_score - score);
- sum_exp = sum_exp * correction + 1.0f;
- vec_scale_add(o_row, v_row, correction, head_dim);
- max_score = score;
- } else {
- float wt = expf(score - max_score);
- sum_exp += wt;
- vec_axpy_inplace(o_row, v_row, wt, head_dim);
- }
- }
-
- if (sum_exp > 0.0f)
- vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim);
- }
-}
-
-#ifdef USE_BLAS
-static void causal_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
- int rows = query_end - query_start;
- int keys = query_end - start;
-
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
-
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden, k,
- kv_hidden, 0.0f, scores, keys);
- for (int r = 0; r < rows; r++) {
- int allowed = query_start + r - start + 1;
- float *row = scores + (size_t)r * keys;
- for (int j = allowed; j < keys; j++)
- row[j] = -INFINITY;
- }
- softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
-}
-#endif
-
-static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
- int task_id = 0;
-#ifdef USE_BLAS
- float *scores = t->scores ? t->scores + (size_t)tid * PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
-#endif
-
- for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
-#ifdef USE_BLAS
- if (scores && end - start >= PACKED_ATTN_BLAS_MIN_SEQ) {
- causal_gqa_attention_packed_blas_rows(t->out, t->Q, t->K, t->V, start, t->n_heads,
- t->n_kv_heads, t->head_dim, t->scale, h, q0,
- q1, scores);
- continue;
- }
-#endif
- causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start, t->n_heads,
- t->n_kv_heads, t->head_dim, t->scale, h, q0,
- q1);
- }
- }
- }
-}
-
-void causal_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * (len + 1) / 2 * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
- float *scores = NULL;
-#ifdef USE_BLAS
- size_t required = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required)
- scores = scratch;
-#else
- (void)scratch;
- (void)scratch_bytes;
-#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
- if (tp_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096)
- parallel_for(packed_causal_gqa_attn_worker, &task);
- else
- packed_causal_gqa_attn_worker(0, 1, &task);
-}
-
-void causal_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale) {
- size_t scratch_bytes = bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
- head_dim, scale, scores, scratch_bytes);
- free(scores);
-}