1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
/*
* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply.
*/
#include "kernels.h"
#include "impl.h"
#include <math.h>
#include <stdlib.h>
#include <string.h>
#if (defined(__AVX512F__) || defined(__AVX2__)) && \
(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
# include <immintrin.h>
#endif
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
# include <arm_neon.h>
#endif
#ifdef USE_BLAS
# ifdef __APPLE__
# include <Accelerate/Accelerate.h>
# else
# include <cblas.h>
# endif
#endif
#ifndef M_PI
# define M_PI 3.14159265358979323846
#endif
/* Position embeddings */
void compute_rope_neox(
float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) {
int half = head_dim / 2;
for (int s = 0; s < seq; s++) {
float pos = (float)positions[s];
for (int d = 0; d < half; d++) {
float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
float angle = pos * freq;
float c = cosf(angle);
float sn = sinf(angle);
/* Duplicate for full head_dim */
cos_out[s * head_dim + d] = c;
cos_out[s * head_dim + half + d] = c;
sin_out[s * head_dim + d] = sn;
sin_out[s * head_dim + half + d] = sn;
}
}
}
void apply_rope_neox(
float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) {
/*
* NeoX split-half style:
* x1 = x[..., :half], x2 = x[..., half:]
* rotated = cat(-x2, x1)
* result = x * cos + rotated * sin
*/
int half = head_dim / 2;
int hidden = n_heads * head_dim;
for (int s = 0; s < seq; s++) {
const float *c = cos_vals + s * head_dim;
const float *sn = sin_vals + s * head_dim;
for (int h = 0; h < n_heads; h++) {
float *vec = x + s * hidden + h * head_dim;
#if defined(__AVX512F__) && defined(__FMA__)
int d = 0;
for (; d + 16 <= half; d += 16) {
__m512 x1 = _mm512_loadu_ps(vec + d);
__m512 x2 = _mm512_loadu_ps(vec + half + d);
/* RoPE cache duplicates cos/sin across halves. */
__m512 cc = _mm512_loadu_ps(c + d);
__m512 ss = _mm512_loadu_ps(sn + d);
__m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss));
__m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss));
_mm512_storeu_ps(vec + d, new1);
_mm512_storeu_ps(vec + half + d, new2);
}
for (; d < half; d++) {
float x1 = vec[d];
float x2 = vec[half + d];
vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#elif defined(__AVX2__) && defined(__FMA__)
int d = 0;
for (; d + 8 <= half; d += 8) {
__m256 x1 = _mm256_loadu_ps(vec + d);
__m256 x2 = _mm256_loadu_ps(vec + half + d);
__m256 cc = _mm256_loadu_ps(c + d);
__m256 ss = _mm256_loadu_ps(sn + d);
__m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss));
__m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss));
_mm256_storeu_ps(vec + d, new1);
_mm256_storeu_ps(vec + half + d, new2);
}
for (; d < half; d++) {
float x1 = vec[d];
float x2 = vec[half + d];
vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
int d = 0;
for (; d + 4 <= half; d += 4) {
float32x4_t x1 = vld1q_f32(vec + d);
float32x4_t x2 = vld1q_f32(vec + half + d);
float32x4_t cc = vld1q_f32(c + d);
float32x4_t ss = vld1q_f32(sn + d);
float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss);
float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss);
vst1q_f32(vec + d, new1);
vst1q_f32(vec + half + d, new2);
}
for (; d < half; d++) {
float x1 = vec[d];
float x2 = vec[half + d];
vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#else
for (int d = 0; d < half; d++) {
float x1 = vec[d]; /* first half */
float x2 = vec[half + d]; /* second half */
vec[d] = x1 * c[d] + (-x2) * sn[d];
vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
}
#endif
}
}
}
|