aboutsummaryrefslogtreecommitdiff
path: root/src/kernels_rope.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/kernels_rope.c')
-rw-r--r--src/kernels_rope.c137
1 files changed, 137 insertions, 0 deletions
diff --git a/src/kernels_rope.c b/src/kernels_rope.c
new file mode 100644
index 0000000..ef1709a
--- /dev/null
+++ b/src/kernels_rope.c
@@ -0,0 +1,137 @@
+#include "kernels.h"
+#include "kernels_impl.h"
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if (defined(__AVX512F__) || defined(__AVX2__)) && \
+ (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
+#include <immintrin.h>
+#endif
+#if defined(__ARM_NEON) || defined(__ARM_NEON__)
+#include <arm_neon.h>
+#endif
+
+#ifdef USE_BLAS
+#ifdef __APPLE__
+#include <Accelerate/Accelerate.h>
+#else
+#include <cblas.h>
+#endif
+#endif
+
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
+/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */
+
+/* ========================================================================
+ * Position Embeddings
+ * ======================================================================== */
+
+void embed_compute_rope_neox(
+ float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) {
+ int half = head_dim / 2;
+
+ for (int s = 0; s < seq; s++) {
+ float pos = (float)positions[s];
+ for (int d = 0; d < half; d++) {
+ float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
+ float angle = pos * freq;
+ float c = cosf(angle);
+ float sn = sinf(angle);
+ /* Duplicate for full head_dim */
+ cos_out[s * head_dim + d] = c;
+ cos_out[s * head_dim + half + d] = c;
+ sin_out[s * head_dim + d] = sn;
+ sin_out[s * head_dim + half + d] = sn;
+ }
+ }
+}
+
+void embed_apply_rope_neox(
+ float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) {
+ /*
+ * NeoX split-half style:
+ * x1 = x[..., :half], x2 = x[..., half:]
+ * rotated = cat(-x2, x1)
+ * result = x * cos + rotated * sin
+ */
+ int half = head_dim / 2;
+ int hidden = n_heads * head_dim;
+
+ for (int s = 0; s < seq; s++) {
+ const float *c = cos_vals + s * head_dim;
+ const float *sn = sin_vals + s * head_dim;
+
+ for (int h = 0; h < n_heads; h++) {
+ float *vec = x + s * hidden + h * head_dim;
+
+#if defined(__AVX512F__) && defined(__FMA__)
+ int d = 0;
+ for (; d + 16 <= half; d += 16) {
+ __m512 x1 = _mm512_loadu_ps(vec + d);
+ __m512 x2 = _mm512_loadu_ps(vec + half + d);
+ /* RoPE cache duplicates cos/sin across halves. */
+ __m512 cc = _mm512_loadu_ps(c + d);
+ __m512 ss = _mm512_loadu_ps(sn + d);
+ __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss));
+ __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss));
+ _mm512_storeu_ps(vec + d, new1);
+ _mm512_storeu_ps(vec + half + d, new2);
+ }
+ for (; d < half; d++) {
+ float x1 = vec[d];
+ float x2 = vec[half + d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#elif defined(__AVX2__) && defined(__FMA__)
+ int d = 0;
+ for (; d + 8 <= half; d += 8) {
+ __m256 x1 = _mm256_loadu_ps(vec + d);
+ __m256 x2 = _mm256_loadu_ps(vec + half + d);
+ __m256 cc = _mm256_loadu_ps(c + d);
+ __m256 ss = _mm256_loadu_ps(sn + d);
+ __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss));
+ __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss));
+ _mm256_storeu_ps(vec + d, new1);
+ _mm256_storeu_ps(vec + half + d, new2);
+ }
+ for (; d < half; d++) {
+ float x1 = vec[d];
+ float x2 = vec[half + d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
+ int d = 0;
+ for (; d + 4 <= half; d += 4) {
+ float32x4_t x1 = vld1q_f32(vec + d);
+ float32x4_t x2 = vld1q_f32(vec + half + d);
+ float32x4_t cc = vld1q_f32(c + d);
+ float32x4_t ss = vld1q_f32(sn + d);
+ float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss);
+ float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss);
+ vst1q_f32(vec + d, new1);
+ vst1q_f32(vec + half + d, new2);
+ }
+ for (; d < half; d++) {
+ float x1 = vec[d];
+ float x2 = vec[half + d];
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#else
+ for (int d = 0; d < half; d++) {
+ float x1 = vec[d]; /* first half */
+ float x2 = vec[half + d]; /* second half */
+ vec[d] = x1 * c[d] + (-x2) * sn[d];
+ vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
+ }
+#endif
+ }
+ }
+}