/* * Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ #include "kernels.h" #include "impl.h" #include #include #include #if (defined(__AVX512F__) || defined(__AVX2__)) && \ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) # include #endif #if defined(__ARM_NEON) || defined(__ARM_NEON__) # include #endif #ifdef USE_BLAS # ifdef __APPLE__ # include # else # include # endif #endif #ifndef M_PI # define M_PI 3.14159265358979323846 #endif /* Position embeddings */ void compute_rope_neox( float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { int half = head_dim / 2; for (int s = 0; s < seq; s++) { float pos = (float)positions[s]; for (int d = 0; d < half; d++) { float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); float angle = pos * freq; float c = cosf(angle); float sn = sinf(angle); /* Duplicate for full head_dim */ cos_out[s * head_dim + d] = c; cos_out[s * head_dim + half + d] = c; sin_out[s * head_dim + d] = sn; sin_out[s * head_dim + half + d] = sn; } } } void apply_rope_neox( float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { /* * NeoX split-half style: * x1 = x[..., :half], x2 = x[..., half:] * rotated = cat(-x2, x1) * result = x * cos + rotated * sin */ int half = head_dim / 2; int hidden = n_heads * head_dim; for (int s = 0; s < seq; s++) { const float *c = cos_vals + s * head_dim; const float *sn = sin_vals + s * head_dim; for (int h = 0; h < n_heads; h++) { float *vec = x + s * hidden + h * head_dim; #if defined(__AVX512F__) && defined(__FMA__) int d = 0; for (; d + 16 <= half; d += 16) { __m512 x1 = _mm512_loadu_ps(vec + d); __m512 x2 = _mm512_loadu_ps(vec + half + d); /* RoPE cache duplicates cos/sin across halves. */ __m512 cc = _mm512_loadu_ps(c + d); __m512 ss = _mm512_loadu_ps(sn + d); __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); _mm512_storeu_ps(vec + d, new1); _mm512_storeu_ps(vec + half + d, new2); } for (; d < half; d++) { float x1 = vec[d]; float x2 = vec[half + d]; vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #elif defined(__AVX2__) && defined(__FMA__) int d = 0; for (; d + 8 <= half; d += 8) { __m256 x1 = _mm256_loadu_ps(vec + d); __m256 x2 = _mm256_loadu_ps(vec + half + d); __m256 cc = _mm256_loadu_ps(c + d); __m256 ss = _mm256_loadu_ps(sn + d); __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); _mm256_storeu_ps(vec + d, new1); _mm256_storeu_ps(vec + half + d, new2); } for (; d < half; d++) { float x1 = vec[d]; float x2 = vec[half + d]; vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #elif defined(__ARM_NEON) || defined(__ARM_NEON__) int d = 0; for (; d + 4 <= half; d += 4) { float32x4_t x1 = vld1q_f32(vec + d); float32x4_t x2 = vld1q_f32(vec + half + d); float32x4_t cc = vld1q_f32(c + d); float32x4_t ss = vld1q_f32(sn + d); float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss); float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss); vst1q_f32(vec + d, new1); vst1q_f32(vec + half + d, new2); } for (; d < half; d++) { float x1 = vec[d]; float x2 = vec[half + d]; vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #else for (int d = 0; d < half; d++) { float x1 = vec[d]; /* first half */ float x2 = vec[half + d]; /* second half */ vec[d] = x1 * c[d] + (-x2) * sn[d]; vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; } #endif } } }