diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-03 11:52:09 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | cb6ae157a09b6794f5d6da27dbc97e0ad791a146 (patch) | |
| tree | 6085cef187a3f326a257394517bf76192c742c2d | |
| parent | 7cc87daedf764da43c338ba1cba739bd6af6e950 (diff) | |
| download | ffwd-cb6ae157a09b6794f5d6da27dbc97e0ad791a146.tar.gz ffwd-cb6ae157a09b6794f5d6da27dbc97e0ad791a146.zip | |
late: add packed maxsim scoring
| -rw-r--r-- | pplx_embed.c | 25 | ||||
| -rw-r--r-- | pplx_embed.h | 12 | ||||
| -rw-r--r-- | qwen_asr_kernels.c | 12 | ||||
| -rw-r--r-- | qwen_asr_kernels.h | 3 |
4 files changed, 45 insertions, 7 deletions
diff --git a/pplx_embed.c b/pplx_embed.c index c4e92a0..35bd739 100644 --- a/pplx_embed.c +++ b/pplx_embed.c @@ -945,9 +945,7 @@ float pplx_late_maxsim(const float *query_vectors, int query_tokens, float best = -FLT_MAX; for (int di = 0; di < doc_tokens; di++) { const float *d = doc_vectors + (size_t)di * dim; - float dot = 0.0f; - for (int k = 0; k < dim; k++) - dot += q[k] * d[k]; + float dot = qwen_dot_f32(q, d, dim); if (dot > best) best = dot; } score += best; @@ -955,6 +953,27 @@ float pplx_late_maxsim(const float *query_vectors, int query_tokens, return score; } +int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens, + const float *doc_vectors, + const int *doc_offsets, int docs, + int dim, float *scores) +{ + if (!query_vectors || !doc_vectors || !doc_offsets || !scores || + query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0) + return -1; + + for (int i = 0; i < docs; i++) { + int start = doc_offsets[i]; + int end = doc_offsets[i + 1]; + if (start < 0 || end <= start) + return -1; + scores[i] = pplx_late_maxsim(query_vectors, query_tokens, + doc_vectors + (size_t)start * dim, + end - start, dim); + } + return 0; +} + size_t pplx_workspace_nbytes(const pplx_workspace_t *ws) { if (!ws) return 0; diff --git a/pplx_embed.h b/pplx_embed.h index 294508f..5a2501b 100644 --- a/pplx_embed.h +++ b/pplx_embed.h @@ -274,6 +274,18 @@ float pplx_late_maxsim(const float *query_vectors, int query_tokens, const float *doc_vectors, int doc_tokens, int dim); +/* + * Score one query against a packed/ragged batch of documents. + * + * doc_offsets has docs + 1 entries. Document i is stored in + * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim). + * scores is caller-provided [docs]. + */ +int pplx_late_maxsim_batch(const float *query_vectors, int query_tokens, + const float *doc_vectors, + const int *doc_offsets, int docs, + int dim, float *scores); + /* Verbose level: 0=quiet, 1=info, 2=debug */ extern int pplx_verbose; extern int qwen_verbose; diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c index c948c03..031a0c5 100644 --- a/qwen_asr_kernels.c +++ b/qwen_asr_kernels.c @@ -1195,7 +1195,11 @@ void qwen_softmax(float *x, int rows, int cols) { * Attention Operations * ======================================================================== */ -static inline float qwen_dot_f32(const float *a, const float *b, int n) { +float qwen_dot_f32(const float *a, const float *b, int n) { + return qwen_dot_f32_impl(a, b, n); +} + +static inline float qwen_dot_f32_fast(const float *a, const float *b, int n) { return qwen_dot_f32_impl(a, b, n); } @@ -1238,7 +1242,7 @@ void qwen_bidirectional_attention(float *out, const float *Q, const float *K, const float *k_row = K + j * hidden + h * head_dim; const float *v_row = V + j * hidden + h * head_dim; - float score = qwen_dot_f32(q_row, k_row, head_dim) * scale; + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); @@ -1288,7 +1292,7 @@ static void qwen_causal_attention_heads(float *out, const float *Q, const float const float *k_row = K + j * kv_hidden + kv_h * head_dim; const float *v_row = V + j * kv_hidden + kv_h * head_dim; - float score = qwen_dot_f32(q_row, k_row, head_dim) * scale; + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); @@ -1378,7 +1382,7 @@ static void qwen_bidirectional_gqa_attention_packed_online_rows( const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim; const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim; - float score = qwen_dot_f32(q_row, k_row, head_dim) * scale; + float score = qwen_dot_f32_fast(q_row, k_row, head_dim) * scale; if (score > max_score) { float correction = expf(max_score - score); diff --git a/qwen_asr_kernels.h b/qwen_asr_kernels.h index c32ec3a..63b704d 100644 --- a/qwen_asr_kernels.h +++ b/qwen_asr_kernels.h @@ -59,6 +59,9 @@ void qwen_linear_nobias_bf16_qkv(float *q, float *k, float *v, const float *x, void qwen_matmul_t_bf16(float *C, const float *A, const uint16_t *B_bf16, int M, int K, int N); +/* Dot product using the best available local SIMD implementation. */ +float qwen_dot_f32(const float *a, const float *b, int n); + /* ======================================================================== * 2D Convolution (for audio encoder conv stem) * ======================================================================== */ |
