aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--qwen_asr_kernels.c53
1 files changed, 48 insertions, 5 deletions
diff --git a/qwen_asr_kernels.c b/qwen_asr_kernels.c
index 031a0c5..6b2e955 100644
--- a/qwen_asr_kernels.c
+++ b/qwen_asr_kernels.c
@@ -935,11 +935,14 @@ void qwen_layer_norm(float *out, const float *x, const float *weight, const floa
}
}
-void qwen_rms_norm(float *out, const float *x, const float *weight,
- int seq_len, int hidden, float eps) {
- for (int s = 0; s < seq_len; s++) {
- const float *x_row = x + s * hidden;
- float *out_row = out + s * hidden;
+#define QWEN_RMS_NORM_PARALLEL_ELEMS 262144
+
+static void qwen_rms_norm_range(float *out, const float *x,
+ const float *weight, int start, int end,
+ int hidden, float eps) {
+ for (int s = start; s < end; s++) {
+ const float *x_row = x + (size_t)s * hidden;
+ float *out_row = out + (size_t)s * hidden;
#if defined(__AVX512F__) && defined(__FMA__)
__m512 accv = _mm512_setzero_ps();
@@ -996,6 +999,46 @@ void qwen_rms_norm(float *out, const float *x, const float *weight,
}
}
+typedef struct {
+ float *out;
+ const float *x;
+ const float *weight;
+ int seq_len;
+ int hidden;
+ float eps;
+} rms_norm_task_t;
+
+static void rms_norm_worker(int tid, int n_threads, void *arg) {
+ rms_norm_task_t *t = (rms_norm_task_t *)arg;
+ int chunk = (t->seq_len + n_threads - 1) / n_threads;
+ int start = tid * chunk;
+ int end = start + chunk;
+ if (end > t->seq_len) end = t->seq_len;
+ if (start >= end) return;
+ qwen_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden,
+ t->eps);
+}
+
+void qwen_rms_norm(float *out, const float *x, const float *weight,
+ int seq_len, int hidden, float eps) {
+ if (seq_len <= 0 || hidden <= 0) return;
+
+ long long elems = (long long)seq_len * hidden;
+ if (tp.n_threads > 1 && elems >= QWEN_RMS_NORM_PARALLEL_ELEMS) {
+ rms_norm_task_t task = {
+ .out = out,
+ .x = x,
+ .weight = weight,
+ .seq_len = seq_len,
+ .hidden = hidden,
+ .eps = eps,
+ };
+ parallel_for(rms_norm_worker, &task);
+ } else {
+ qwen_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps);
+ }
+}
+
void qwen_rms_norm_per_head(float *x, const float *weight,
int seq_len, int n_heads, int head_dim, float eps) {
/* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */