aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-03 11:52:09 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitcb6ae157a09b6794f5d6da27dbc97e0ad791a146 (patch)
tree6085cef187a3f326a257394517bf76192c742c2d
parent7cc87daedf764da43c338ba1cba739bd6af6e950 (diff)
downloadffwd-cb6ae157a09b6794f5d6da27dbc97e0ad791a146.tar.gz
ffwd-cb6ae157a09b6794f5d6da27dbc97e0ad791a146.zip
late: add packed maxsim scoring
-rw-r--r--pplx_embed.c25
-rw-r--r--pplx_embed.h12
-rw-r--r--qwen_asr_kernels.c12
-rw-r--r--qwen_asr_kernels.h3
4 files changed, 45 insertions, 7 deletions
diff --git a/pplx_embed.c b/pplx_embed.c
index c4e92a0..35bd739 100644
--- a/pplx_embed.c
+++ b/pplx_embed.c
@@ -945,9 +945,7 @@ float pplx_late_maxsim(const float *query_vectors, int query_tokens,
float best = -FLT_MAX;
for (int di = 0; di < doc_tokens; di++) {
const float *d = doc_vectors + (size_t)di * dim;
- float dot = 0.0f;
- for (int k = 0; k < dim; k++)
- dot += q[k] * d[k];
+ float dot = qwen_dot_f32(q, d, dim);
if (dot > best) best = dot;
}
score += best;
@@ -955,6 +953,27 @@ float pplx_late_maxsim(const float *query_vectors, int query_tokens,
return score;
}
+int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets, int docs,
+ int dim, float *scores)
+{
+ if (!query_vectors || !doc_vectors || !doc_offsets || !scores ||
+ query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
+ return -1;
+
+ for (int i = 0; i < docs; i++) {
+ int start = doc_offsets[i];
+ int end = doc_offsets[i + 1];
+ if (start < 0 || end <= start)
+ return -1;
+ scores[i] = pplx_late_maxsim(query_vectors, query_tokens,
+ doc_vectors + (size_t)start * dim,
+ end - start, dim);
+ }
+ return 0;
+}
+
size_t pplx_workspace_nbytes(const pplx_workspace_t *ws)
{
if (!ws) return 0;
diff --git a/pplx_embed.h b/pplx_embed.h
index 294508f..5a2501b 100644
--- a/pplx_embed.h
+++ b/pplx_embed.h
@@ -274,6 +274,18 @@ float pplx_late_maxsim(const float *query_vectors, int query_tokens,
const float *doc_vectors, int doc_tokens,
int dim);
+/*
+ * Score one query against a packed/ragged batch of documents.
+ *
+ * doc_offsets has docs + 1 entries. Document i is stored in
+ * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim).
+ * scores is caller-provided [docs].
+ */
+int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets, int docs,
+ int dim, float *scores);
+
/* Verbose level: 0=quiet, 1=info, 2=debug */
extern int pplx_verbose;
extern int qwen_verbose;
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index c948c03..031a0c5 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -1195,7 +1195,11 @@ void qwen_softmax(float *x, int rows, int cols) {
* Attention Operations
* ======================================================================== */
-static inline float qwen_dot_f32(const float *a, const float *b, int n) {
+float qwen_dot_f32(const float *a, const float *b, int n) {
+ return qwen_dot_f32_impl(a, b, n);
+}
+
+static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) {
return qwen_dot_f32_impl(a, b, n);
}
@@ -1238,7 +1242,7 @@ void qwen_bidirectional_attention(float *out, const float *Q, const float *K,
const float *k_row = K + j * hidden + h * head_dim;
const float *v_row = V + j * hidden + h * head_dim;
- float score = qwen_dot_f32(q_row, k_row, head_dim) * scale;
+ float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale;
if (score > max_score) {
float correction = expf(max_score - score);
@@ -1288,7 +1292,7 @@ static void qwen_causal_attention_heads(float *out, const float *Q, const float
const float *k_row = K + j * kv_hidden + kv_h * head_dim;
const float *v_row = V + j * kv_hidden + kv_h * head_dim;
- float score = qwen_dot_f32(q_row, k_row, head_dim) * scale;
+ float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale;
if (score > max_score) {
float correction = expf(max_score - score);
@@ -1378,7 +1382,7 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows(
const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
- float score = qwen_dot_f32(q_row, k_row, head_dim) * scale;
+ float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale;
if (score > max_score) {
float correction = expf(max_score - score);
diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h
index c32ec3a..63b704d 100644
--- a/qwen_asr_kernels.h
+++ b/qwen_asr_kernels.h
@@ -59,6 +59,9 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x,
void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16,
int M, int K, int N);
+/* Dot product using the best available local SIMD implementation. */
+float qwen_dot_f32(const float *a, const float *b, int n);
+
/* ========================================================================
* 2D Convolution (for audio encoder conv stem)
* ======================================================================== */