diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 19:10:59 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 19:10:59 +0200 |
| commit | 7500042ac2c33e0501e3d2ebae5a423e61cc39d1 (patch) | |
| tree | 8841d3eaa30824b1026d0d6a100b7ee2844a5651 /src/kernels_rope.c | |
| parent | 0b36622b697db9a47db0354d8c594fa225da3dfc (diff) | |
| download | ffwd-7500042ac2c33e0501e3d2ebae5a423e61cc39d1.tar.gz ffwd-7500042ac2c33e0501e3d2ebae5a423e61cc39d1.zip | |
kernels: split kernels.c into gemm/norm/act/attn/rope by kind
Diffstat (limited to 'src/kernels_rope.c')
| -rw-r--r-- | src/kernels_rope.c | 137 |
1 files changed, 137 insertions, 0 deletions
diff --git a/src/kernels_rope.c b/src/kernels_rope.c new file mode 100644 index 0000000..ef1709a --- /dev/null +++ b/src/kernels_rope.c @@ -0,0 +1,137 @@ +#include "kernels.h" +#include "kernels_impl.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#if (defined(__AVX512F__) || defined(__AVX2__)) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) +#include <immintrin.h> +#endif +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#include <arm_neon.h> +#endif + +#ifdef USE_BLAS +#ifdef __APPLE__ +#include <Accelerate/Accelerate.h> +#else +#include <cblas.h> +#endif +#endif + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */ + +/* ======================================================================== + * Position Embeddings + * ======================================================================== */ + +void embed_compute_rope_neox( + float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) { + int half = head_dim / 2; + + for (int s = 0; s < seq; s++) { + float pos = (float)positions[s]; + for (int d = 0; d < half; d++) { + float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim); + float angle = pos * freq; + float c = cosf(angle); + float sn = sinf(angle); + /* Duplicate for full head_dim */ + cos_out[s * head_dim + d] = c; + cos_out[s * head_dim + half + d] = c; + sin_out[s * head_dim + d] = sn; + sin_out[s * head_dim + half + d] = sn; + } + } +} + +void embed_apply_rope_neox( + float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) { + /* + * NeoX split-half style: + * x1 = x[..., :half], x2 = x[..., half:] + * rotated = cat(-x2, x1) + * result = x * cos + rotated * sin + */ + int half = head_dim / 2; + int hidden = n_heads * head_dim; + + for (int s = 0; s < seq; s++) { + const float *c = cos_vals + s * head_dim; + const float *sn = sin_vals + s * head_dim; + + for (int h = 0; h < n_heads; h++) { + float *vec = x + s * hidden + h * head_dim; + +#if defined(__AVX512F__) && defined(__FMA__) + int d = 0; + for (; d + 16 <= half; d += 16) { + __m512 x1 = _mm512_loadu_ps(vec + d); + __m512 x2 = _mm512_loadu_ps(vec + half + d); + /* RoPE cache duplicates cos/sin across halves. */ + __m512 cc = _mm512_loadu_ps(c + d); + __m512 ss = _mm512_loadu_ps(sn + d); + __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss)); + __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss)); + _mm512_storeu_ps(vec + d, new1); + _mm512_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__AVX2__) && defined(__FMA__) + int d = 0; + for (; d + 8 <= half; d += 8) { + __m256 x1 = _mm256_loadu_ps(vec + d); + __m256 x2 = _mm256_loadu_ps(vec + half + d); + __m256 cc = _mm256_loadu_ps(c + d); + __m256 ss = _mm256_loadu_ps(sn + d); + __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss)); + __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss)); + _mm256_storeu_ps(vec + d, new1); + _mm256_storeu_ps(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) + int d = 0; + for (; d + 4 <= half; d += 4) { + float32x4_t x1 = vld1q_f32(vec + d); + float32x4_t x2 = vld1q_f32(vec + half + d); + float32x4_t cc = vld1q_f32(c + d); + float32x4_t ss = vld1q_f32(sn + d); + float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss); + float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss); + vst1q_f32(vec + d, new1); + vst1q_f32(vec + half + d, new2); + } + for (; d < half; d++) { + float x1 = vec[d]; + float x2 = vec[half + d]; + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#else + for (int d = 0; d < half; d++) { + float x1 = vec[d]; /* first half */ + float x2 = vec[half + d]; /* second half */ + vec[d] = x1 * c[d] + (-x2) * sn[d]; + vec[half + d] = x2 * c[half + d] + x1 * sn[half + d]; + } +#endif + } + } +} |
