diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/kernels.c | 555 | ||||
| -rw-r--r-- | src/kernels.h | 67 | ||||
| -rw-r--r-- | src/pyseg.c | 503 | ||||
| -rw-r--r-- | src/pyseg.h | 215 | ||||
| -rw-r--r-- | src/pyseg_cli.c | 176 | ||||
| -rw-r--r-- | src/slide.c | 423 | ||||
| -rw-r--r-- | src/wav.c | 127 | ||||
| -rw-r--r-- | src/wav.h | 22 | ||||
| -rw-r--r-- | src/weights.c | 189 | ||||
| -rw-r--r-- | src/weights.h | 49 |
10 files changed, 2326 insertions, 0 deletions
diff --git a/src/kernels.c b/src/kernels.c new file mode 100644 index 0000000..01c1d19 --- /dev/null +++ b/src/kernels.c @@ -0,0 +1,555 @@ +/* Scalar and SIMD implementations of the dense primitive. + * + * Each variant blocks a rectangle of output positions against weight rows and + * accumulates along the window, and the rectangle is chosen by the register + * file rather than by the algorithm. AArch64 has 32 vector registers, so four + * positions by four rows fits: sixteen accumulators plus eight live operands, + * eight loads for sixteen multiply-adds. x86-64 has 16, so the same rectangle + * spills its accumulators to the stack every iteration and four by two is + * faster despite doing half as much per pass; on one server core that + * difference was a factor of 1.8 over a whole recording. + * + * Matrix-vector shapes, meaning the LSTM recurrence where there is only one + * output position, widen instead to eight weight rows. With a single + * activation window there is nothing to reuse but that window, so the weight + * loads are the entire cost and the only thing worth amortising is the + * activation. + * + * Accumulation order differs between the variants, so their results differ in + * the last bits; tests/test_kernels.c bounds that. + */ + +#include "kernels.h" + +#include <math.h> +#include <stddef.h> + +#if defined(__GNUC__) || defined(__clang__) +# define PYSEG_INLINE static inline __attribute__((always_inline)) +#else +# define PYSEG_INLINE static inline +#endif + +static inline int min_int(int a, int b) { return a < b ? a : b; } + +/* ---- portable C ------------------------------------------------------ */ + +PYSEG_INLINE void +dot4x4_scalar(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) { + float acc[4][4] = {{0.0f}}; + for (int j = 0; j < k; ++j) { + const float a0 = a[j], a1 = a[lda + j], a2 = a[2 * lda + j], a3 = a[3 * lda + j]; + for (int i = 0; i < 4; ++i) { + const float wv = w[(size_t)i * k + j]; + acc[0][i] += a0 * wv; + acc[1][i] += a1 * wv; + acc[2][i] += a2 * wv; + acc[3][i] += a3 * wv; + } + } + for (int s = 0; s < 4; ++s) + for (int i = 0; i < 4; ++i) + y[(size_t)s * ldy + i] = bias ? bias[i] + acc[s][i] : acc[s][i]; +} + +/* One output position against however many weight rows are left. */ +PYSEG_INLINE void +row_scalar(float *y, const float *a, const float *w, const float *bias, int n, int k) { + for (int i = 0; i < n; ++i) { + const float *wr = w + (size_t)i * k; + float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; + int j = 0; + for (; j + 4 <= k; j += 4) { + s0 += a[j + 0] * wr[j + 0]; + s1 += a[j + 1] * wr[j + 1]; + s2 += a[j + 2] * wr[j + 2]; + s3 += a[j + 3] * wr[j + 3]; + } + const float s = (s0 + s1) + (s2 + s3); + y[i] = bias ? bias[i] + s : s; + } +} + +static void gemm_nt_scalar(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k) { + for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) { + const int mend = min_int(m0 + PYSEG_MPANEL, m); + const int m4 = m0 + (mend - m0) / 4 * 4; + int i = 0; + for (; i + 4 <= n; i += 4) + for (int s = m0; s < m4; s += 4) + dot4x4_scalar(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, + w + (size_t)i * k, bias ? bias + i : NULL, k); + for (int s = m0; s < m4; ++s) + if (i < n) + row_scalar(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k, + bias ? bias + i : NULL, n - i, k); + for (int s = m4; s < mend; ++s) + row_scalar(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k); + } +} + +static void lstm_cell_scalar(float *h, float *c, const float *rec, const float *in, int hid) { + for (int i = 0; i < hid; ++i) { + const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i]))); + const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i]))); + const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]); + const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i]))); + const float cn = fg * c[i] + ig * gg; + c[i] = cn; + h[i] = og * tanhf(cn); + } +} + +static const pyseg_kernels kernels_scalar = {"scalar", gemm_nt_scalar, lstm_cell_scalar}; + +const pyseg_kernels *pyseg_kernels_scalar(void) { return &kernels_scalar; } + +/* ---- AArch64 NEON ---------------------------------------------------- */ + +#if defined(__aarch64__) && defined(__ARM_NEON) +# define PYSEG_HAVE_SIMD 1 +# include <arm_neon.h> + +/* Four lane-wise accumulators reduced to one vector holding their four totals. + * Pairwise adds do this in three instructions instead of four cross-lane + * reductions plus the moves back into a vector. */ +PYSEG_INLINE float32x4_t hsum4(float32x4_t a, float32x4_t b, float32x4_t c, float32x4_t d) { + return vpaddq_f32(vpaddq_f32(a, b), vpaddq_f32(c, d)); +} + +PYSEG_INLINE void +dot4x4_neon(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) { + const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda; + const float *w0 = w, *w1 = w + k, *w2 = w + 2 * k, *w3 = w + 3 * k; + + float32x4_t c00 = vdupq_n_f32(0.0f), c01 = c00, c02 = c00, c03 = c00; + float32x4_t c10 = c00, c11 = c00, c12 = c00, c13 = c00; + float32x4_t c20 = c00, c21 = c00, c22 = c00, c23 = c00; + float32x4_t c30 = c00, c31 = c00, c32 = c00, c33 = c00; + + for (int j = 0; j < k; j += 4) { + const float32x4_t x0 = vld1q_f32(a0 + j); + const float32x4_t x1 = vld1q_f32(a1 + j); + const float32x4_t x2 = vld1q_f32(a2 + j); + const float32x4_t x3 = vld1q_f32(a3 + j); + const float32x4_t v0 = vld1q_f32(w0 + j); + const float32x4_t v1 = vld1q_f32(w1 + j); + const float32x4_t v2 = vld1q_f32(w2 + j); + const float32x4_t v3 = vld1q_f32(w3 + j); + c00 = vfmaq_f32(c00, x0, v0); + c01 = vfmaq_f32(c01, x0, v1); + c02 = vfmaq_f32(c02, x0, v2); + c03 = vfmaq_f32(c03, x0, v3); + c10 = vfmaq_f32(c10, x1, v0); + c11 = vfmaq_f32(c11, x1, v1); + c12 = vfmaq_f32(c12, x1, v2); + c13 = vfmaq_f32(c13, x1, v3); + c20 = vfmaq_f32(c20, x2, v0); + c21 = vfmaq_f32(c21, x2, v1); + c22 = vfmaq_f32(c22, x2, v2); + c23 = vfmaq_f32(c23, x2, v3); + c30 = vfmaq_f32(c30, x3, v0); + c31 = vfmaq_f32(c31, x3, v1); + c32 = vfmaq_f32(c32, x3, v2); + c33 = vfmaq_f32(c33, x3, v3); + } + + const float32x4_t b = bias ? vld1q_f32(bias) : vdupq_n_f32(0.0f); + vst1q_f32(y, vaddq_f32(b, hsum4(c00, c01, c02, c03))); + vst1q_f32(y + ldy, vaddq_f32(b, hsum4(c10, c11, c12, c13))); + vst1q_f32(y + 2 * ldy, vaddq_f32(b, hsum4(c20, c21, c22, c23))); + vst1q_f32(y + 3 * ldy, vaddq_f32(b, hsum4(c30, c31, c32, c33))); +} + +/* Single output position, eight weight rows at a time. + * + * With one activation window there is nothing to reuse but the window itself, + * so this shape costs one weight load per multiply-add and runs at whatever + * rate the memory system delivers weights, not at the multiply pipes' rate. + * Eight rows measured fastest here: four leaves the load pipeline too shallow, + * and sixteen has enough concurrent streams to start working against the + * hardware prefetcher. */ +PYSEG_INLINE void +row_neon(float *y, const float *a, const float *w, const float *bias, int n, int k) { + int i = 0; + for (; i + 8 <= n; i += 8) { + const float *w0 = w + (size_t)i * k; + float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0; + float32x4_t c4 = c0, c5 = c0, c6 = c0, c7 = c0; + for (int j = 0; j < k; j += 4) { + const float32x4_t x = vld1q_f32(a + j); + c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j)); + c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j)); + c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j)); + c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j)); + c4 = vfmaq_f32(c4, x, vld1q_f32(w0 + 4 * k + j)); + c5 = vfmaq_f32(c5, x, vld1q_f32(w0 + 5 * k + j)); + c6 = vfmaq_f32(c6, x, vld1q_f32(w0 + 6 * k + j)); + c7 = vfmaq_f32(c7, x, vld1q_f32(w0 + 7 * k + j)); + } + const float32x4_t lo = hsum4(c0, c1, c2, c3); + const float32x4_t hi = hsum4(c4, c5, c6, c7); + if (bias) { + vst1q_f32(y + i, vaddq_f32(vld1q_f32(bias + i), lo)); + vst1q_f32(y + i + 4, vaddq_f32(vld1q_f32(bias + i + 4), hi)); + } else { + vst1q_f32(y + i, lo); + vst1q_f32(y + i + 4, hi); + } + } + for (; i + 4 <= n; i += 4) { + const float *w0 = w + (size_t)i * k; + float32x4_t c0 = vdupq_n_f32(0.0f), c1 = c0, c2 = c0, c3 = c0; + for (int j = 0; j < k; j += 4) { + const float32x4_t x = vld1q_f32(a + j); + c0 = vfmaq_f32(c0, x, vld1q_f32(w0 + j)); + c1 = vfmaq_f32(c1, x, vld1q_f32(w0 + k + j)); + c2 = vfmaq_f32(c2, x, vld1q_f32(w0 + 2 * k + j)); + c3 = vfmaq_f32(c3, x, vld1q_f32(w0 + 3 * k + j)); + } + const float32x4_t v = hsum4(c0, c1, c2, c3); + vst1q_f32(y + i, bias ? vaddq_f32(vld1q_f32(bias + i), v) : v); + } + for (; i < n; ++i) { + const float *wr = w + (size_t)i * k; + float32x4_t c0 = vdupq_n_f32(0.0f); + for (int j = 0; j < k; j += 4) + c0 = vfmaq_f32(c0, vld1q_f32(a + j), vld1q_f32(wr + j)); + const float s = vaddvq_f32(c0); + y[i] = bias ? bias[i] + s : s; + } +} + +static void gemm_nt_neon(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k) { + if (m == 1) { + row_neon(y, a, w, bias, n, k); + return; + } + for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) { + const int mend = min_int(m0 + PYSEG_MPANEL, m); + const int m4 = m0 + (mend - m0) / 4 * 4; + int i = 0; + for (; i + 4 <= n; i += 4) + for (int s = m0; s < m4; s += 4) + dot4x4_neon(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k, + bias ? bias + i : NULL, k); + for (int s = m0; s < m4; ++s) + if (i < n) + row_neon(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k, + bias ? bias + i : NULL, n - i, k); + for (int s = m4; s < mend; ++s) + row_neon(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k); + } +} + +/* Vector exponential, the Cephes single-precision scheme: split the argument + * into an integer power of two and a remainder inside half a ln 2, evaluate a + * degree-six polynomial on the remainder, then scale by assembling the power + * of two directly in the exponent field. Accurate to about one unit in the + * last place, which is the same claim libm makes. + * + * The clamp is what keeps the exponent assembly in range; beyond it the + * logistic and hyperbolic functions built on this are saturated anyway. */ +PYSEG_INLINE float32x4_t vexpq(float32x4_t x) { + x = vminq_f32(vmaxq_f32(x, vdupq_n_f32(-88.0f)), vdupq_n_f32(88.0f)); + + const float32x4_t n = vrndnq_f32(vmulq_f32(x, vdupq_n_f32(1.44269504088896341f))); + /* ln 2 split in two so the product with n stays exact in the high part. */ + float32x4_t r = vfmsq_f32(x, n, vdupq_n_f32(0.693359375f)); + r = vfmsq_f32(r, n, vdupq_n_f32(-2.12194440e-4f)); + + float32x4_t p = vdupq_n_f32(1.9875691500e-4f); + p = vfmaq_f32(vdupq_n_f32(1.3981999507e-3f), p, r); + p = vfmaq_f32(vdupq_n_f32(8.3334519073e-3f), p, r); + p = vfmaq_f32(vdupq_n_f32(4.1665795894e-2f), p, r); + p = vfmaq_f32(vdupq_n_f32(1.6666665459e-1f), p, r); + p = vfmaq_f32(vdupq_n_f32(5.0000001201e-1f), p, r); + p = vfmaq_f32(vaddq_f32(r, vdupq_n_f32(1.0f)), p, vmulq_f32(r, r)); + + const int32x4_t bits = vshlq_n_s32(vaddq_s32(vcvtq_s32_f32(n), vdupq_n_s32(127)), 23); + return vmulq_f32(p, vreinterpretq_f32_s32(bits)); +} + +PYSEG_INLINE float32x4_t vsigmoidq(float32x4_t x) { + return vdivq_f32(vdupq_n_f32(1.0f), vaddq_f32(vdupq_n_f32(1.0f), vexpq(vnegq_f32(x)))); +} + +/* tanh x = 2 sigmoid(2x) - 1. The subtraction loses relative accuracy near + * zero, where sigmoid is close to a half, but the absolute error stays at the + * rounding level of the result and that is what propagates into the cell + * state. */ +PYSEG_INLINE float32x4_t vtanhq(float32x4_t x) { + return vfmsq_f32(vdupq_n_f32(-1.0f), vsigmoidq(vaddq_f32(x, x)), vdupq_n_f32(-2.0f)); +} + +static void lstm_cell_neon(float *h, float *c, const float *rec, const float *in, int hid) { + for (int i = 0; i < hid; i += 4) { + const float32x4_t ig = vsigmoidq(vaddq_f32(vld1q_f32(rec + i), vld1q_f32(in + i))); + const float32x4_t fg = + vsigmoidq(vaddq_f32(vld1q_f32(rec + hid + i), vld1q_f32(in + hid + i))); + const float32x4_t gg = + vtanhq(vaddq_f32(vld1q_f32(rec + 2 * hid + i), vld1q_f32(in + 2 * hid + i))); + const float32x4_t og = + vsigmoidq(vaddq_f32(vld1q_f32(rec + 3 * hid + i), vld1q_f32(in + 3 * hid + i))); + const float32x4_t cn = vfmaq_f32(vmulq_f32(ig, gg), fg, vld1q_f32(c + i)); + vst1q_f32(c + i, cn); + vst1q_f32(h + i, vmulq_f32(og, vtanhq(cn))); + } +} + +static const pyseg_kernels kernels_simd = {"neon", gemm_nt_neon, lstm_cell_neon}; +#endif /* aarch64 NEON */ + +/* ---- x86-64 AVX2 ----------------------------------------------------- + * + * Compiled into every x86 build whatever baseline the rest of the file uses, + * by carrying the instruction set on the functions rather than on the command + * line, and selected at startup against the running processor. One binary is + * then both safe on a host without AVX2 and fast on one with it. + * + * Every function reachable from an AVX2 entry point needs the same attribute, + * including the always-inlined ones, or the compiler refuses to inline across + * the mismatch. */ + +#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) +# define PYSEG_HAVE_SIMD 1 +# define PYSEG_X86_DISPATCH 1 +# include <immintrin.h> + +# define PYSEG_AVX2 __attribute__((target("avx2,fma"))) +# define PYSEG_AVX2_INLINE static inline __attribute__((always_inline, target("avx2,fma"))) + +PYSEG_AVX2_INLINE float hsum256(__m256 v) { + __m128 lo = _mm256_castps256_ps128(v); + __m128 hi = _mm256_extractf128_ps(v, 1); + lo = _mm_add_ps(lo, hi); + lo = _mm_add_ps(lo, _mm_movehl_ps(lo, lo)); + lo = _mm_add_ss(lo, _mm_shuffle_ps(lo, lo, 0x55)); + return _mm_cvtss_f32(lo); +} + +/* Four output positions against two weight rows: eight accumulators, which is + * half the register file. That is the shape this instruction set allows. The + * NEON kernel above blocks four by four because it has thirty-two vector + * registers to hold sixteen accumulators plus the operands; asking for the + * same here spills the accumulators to the stack every iteration and costs + * more than the blocking saves. + * + * The window length is a multiple of four but not always of eight, so the + * eight-wide body is followed by at most one four-wide step, done in scalar + * code since it is at most four elements of at least sixty. */ +PYSEG_AVX2_INLINE void +dot4x2_avx2(float *y, int ldy, const float *a, int lda, const float *w, const float *bias, int k) { + const float *a0 = a, *a1 = a + lda, *a2 = a + 2 * lda, *a3 = a + 3 * lda; + const float *w0 = w, *w1 = w + k; + + __m256 c00 = _mm256_setzero_ps(), c01 = c00, c10 = c00, c11 = c00; + __m256 c20 = c00, c21 = c00, c30 = c00, c31 = c00; + + int j = 0; + for (; j + 8 <= k; j += 8) { + const __m256 v0 = _mm256_loadu_ps(w0 + j); + const __m256 v1 = _mm256_loadu_ps(w1 + j); + const __m256 x0 = _mm256_loadu_ps(a0 + j); + const __m256 x1 = _mm256_loadu_ps(a1 + j); + const __m256 x2 = _mm256_loadu_ps(a2 + j); + const __m256 x3 = _mm256_loadu_ps(a3 + j); + c00 = _mm256_fmadd_ps(x0, v0, c00); + c01 = _mm256_fmadd_ps(x0, v1, c01); + c10 = _mm256_fmadd_ps(x1, v0, c10); + c11 = _mm256_fmadd_ps(x1, v1, c11); + c20 = _mm256_fmadd_ps(x2, v0, c20); + c21 = _mm256_fmadd_ps(x2, v1, c21); + c30 = _mm256_fmadd_ps(x3, v0, c30); + c31 = _mm256_fmadd_ps(x3, v1, c31); + } + + const float *ar[4] = {a0, a1, a2, a3}; + __m256 acc[4][2] = {{c00, c01}, {c10, c11}, {c20, c21}, {c30, c31}}; + for (int s = 0; s < 4; ++s) + for (int i = 0; i < 2; ++i) { + float v = hsum256(acc[s][i]); + for (int t = j; t < k; ++t) + v += ar[s][t] * w[(size_t)i * k + t]; + y[(size_t)s * ldy + i] = bias ? bias[i] + v : v; + } +} + +/* Single output position, eight weight rows at a time; the matrix-vector shape + * of the LSTM recurrence. Eight accumulators fit, and one activation load then + * serves eight weight loads. */ +PYSEG_AVX2_INLINE void +row_avx2(float *y, const float *a, const float *w, const float *bias, int n, int k) { + int i = 0; + for (; i + 8 <= n; i += 8) { + const float *w0 = w + (size_t)i * k; + __m256 c0 = _mm256_setzero_ps(), c1 = c0, c2 = c0, c3 = c0; + __m256 c4 = c0, c5 = c0, c6 = c0, c7 = c0; + int j = 0; + for (; j + 8 <= k; j += 8) { + const __m256 x = _mm256_loadu_ps(a + j); + const float *p = w0 + j; + c0 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p), c0); + c1 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + k), c1); + c2 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 2 * k), c2); + c3 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 3 * k), c3); + c4 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 4 * k), c4); + c5 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 5 * k), c5); + c6 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 6 * k), c6); + c7 = _mm256_fmadd_ps(x, _mm256_loadu_ps(p + 7 * k), c7); + } + const __m256 acc[8] = {c0, c1, c2, c3, c4, c5, c6, c7}; + for (int q = 0; q < 8; ++q) { + float v = hsum256(acc[q]); + for (int t = j; t < k; ++t) + v += a[t] * w0[(size_t)q * k + t]; + y[i + q] = bias ? bias[i + q] + v : v; + } + } + for (; i < n; ++i) { + const float *wr = w + (size_t)i * k; + __m256 acc = _mm256_setzero_ps(); + int j = 0; + for (; j + 8 <= k; j += 8) + acc = _mm256_fmadd_ps(_mm256_loadu_ps(a + j), _mm256_loadu_ps(wr + j), acc); + float v = hsum256(acc); + for (; j < k; ++j) + v += a[j] * wr[j]; + y[i] = bias ? bias[i] + v : v; + } +} + +PYSEG_AVX2 static void gemm_nt_avx2(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k) { + if (m == 1) { + row_avx2(y, a, w, bias, n, k); + return; + } + for (int m0 = 0; m0 < m; m0 += PYSEG_MPANEL) { + const int mend = min_int(m0 + PYSEG_MPANEL, m); + const int m4 = m0 + (mend - m0) / 4 * 4; + int i = 0; + for (; i + 2 <= n; i += 2) + for (int s = m0; s < m4; s += 4) + dot4x2_avx2(y + (size_t)s * ldy + i, ldy, a + (size_t)s * lda, lda, w + (size_t)i * k, + bias ? bias + i : NULL, k); + for (int s = m0; s < m4; ++s) + if (i < n) + row_avx2(y + (size_t)s * ldy + i, a + (size_t)s * lda, w + (size_t)i * k, + bias ? bias + i : NULL, n - i, k); + for (int s = m4; s < mend; ++s) + row_avx2(y + (size_t)s * ldy, a + (size_t)s * lda, w, bias, n, k); + } +} + +/* The same Cephes scheme as the NEON path; see the comment there. */ +PYSEG_AVX2_INLINE __m256 vexp256(__m256 x) { + x = _mm256_min_ps(_mm256_max_ps(x, _mm256_set1_ps(-88.0f)), _mm256_set1_ps(88.0f)); + + const __m256 n = _mm256_round_ps(_mm256_mul_ps(x, _mm256_set1_ps(1.44269504088896341f)), + _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + __m256 r = _mm256_fnmadd_ps(n, _mm256_set1_ps(0.693359375f), x); + r = _mm256_fnmadd_ps(n, _mm256_set1_ps(-2.12194440e-4f), r); + + __m256 p = _mm256_set1_ps(1.9875691500e-4f); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.3981999507e-3f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(8.3334519073e-3f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(4.1665795894e-2f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(1.6666665459e-1f)); + p = _mm256_fmadd_ps(p, r, _mm256_set1_ps(5.0000001201e-1f)); + p = _mm256_fmadd_ps(p, _mm256_mul_ps(r, r), _mm256_add_ps(r, _mm256_set1_ps(1.0f))); + + const __m256i bits = + _mm256_slli_epi32(_mm256_add_epi32(_mm256_cvtps_epi32(n), _mm256_set1_epi32(127)), 23); + return _mm256_mul_ps(p, _mm256_castsi256_ps(bits)); +} + +PYSEG_AVX2_INLINE __m256 vsigmoid256(__m256 x) { + const __m256 one = _mm256_set1_ps(1.0f); + return _mm256_div_ps(one, _mm256_add_ps(one, vexp256(_mm256_sub_ps(_mm256_setzero_ps(), x)))); +} + +PYSEG_AVX2_INLINE __m256 vtanh256(__m256 x) { + return _mm256_fmsub_ps(vsigmoid256(_mm256_add_ps(x, x)), _mm256_set1_ps(2.0f), + _mm256_set1_ps(1.0f)); +} + +PYSEG_AVX2_INLINE __m256 gate256(const float *rec, const float *in, int offset) { + return _mm256_add_ps(_mm256_loadu_ps(rec + offset), _mm256_loadu_ps(in + offset)); +} + +PYSEG_AVX2 static void +lstm_cell_avx2(float *h, float *c, const float *rec, const float *in, int hid) { + int i = 0; + for (; i + 8 <= hid; i += 8) { + const __m256 ig = vsigmoid256(gate256(rec, in, i)); + const __m256 fg = vsigmoid256(gate256(rec, in, hid + i)); + const __m256 gg = vtanh256(gate256(rec, in, 2 * hid + i)); + const __m256 og = vsigmoid256(gate256(rec, in, 3 * hid + i)); + const __m256 cn = _mm256_fmadd_ps(fg, _mm256_loadu_ps(c + i), _mm256_mul_ps(ig, gg)); + _mm256_storeu_ps(c + i, cn); + _mm256_storeu_ps(h + i, _mm256_mul_ps(og, vtanh256(cn))); + } + for (; i < hid; ++i) { + const float ig = 1.0f / (1.0f + expf(-(rec[i] + in[i]))); + const float fg = 1.0f / (1.0f + expf(-(rec[hid + i] + in[hid + i]))); + const float gg = tanhf(rec[2 * hid + i] + in[2 * hid + i]); + const float og = 1.0f / (1.0f + expf(-(rec[3 * hid + i] + in[3 * hid + i]))); + const float cn = fg * c[i] + ig * gg; + c[i] = cn; + h[i] = og * tanhf(cn); + } +} + +static const pyseg_kernels kernels_avx2 = {"avx2", gemm_nt_avx2, lstm_cell_avx2}; + +/* AVX2 also needs the operating system to have enabled the wider register + * state, which __builtin_cpu_supports checks along with the feature bits. */ +static int cpu_has_avx2(void) { +# if defined(__has_builtin) +# if __has_builtin(__builtin_cpu_init) + __builtin_cpu_init(); +# endif +# elif defined(__GNUC__) + __builtin_cpu_init(); +# endif + return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma"); +} +#endif /* x86 */ + +const pyseg_kernels *pyseg_kernels_simd(void) { +#if defined(PYSEG_X86_DISPATCH) + /* Resolved once; the answer cannot change while the process runs. */ + static const pyseg_kernels *chosen; + if (!chosen) + chosen = cpu_has_avx2() ? &kernels_avx2 : &kernels_scalar; + return chosen; +#elif defined(PYSEG_HAVE_SIMD) + return &kernels_simd; +#else + return &kernels_scalar; +#endif +} diff --git a/src/kernels.h b/src/kernels.h new file mode 100644 index 0000000..678b844 --- /dev/null +++ b/src/kernels.h @@ -0,0 +1,67 @@ +/* The single dense primitive behind the whole forward pass. + * + * Every layer of this model computes, for a run of output positions, the dot + * products of a window of activations against a fixed set of weight rows. The + * convolutions reach that form because activations are stored time-major: the + * taps a kernel-5 convolution needs for one output step are five consecutive + * frames, which is one contiguous span, and the next output step starts one + * frame later. The sinc filterbank is the same thing with a span of 251 + * samples advancing ten samples at a time. So the only parameter that changes + * between layers is how far apart consecutive windows start, which is what + * lda is; it is unrelated to the window length k. + * + * Taking every output position in one call is what makes this fast. A single + * position would re-read the whole weight matrix; taking four at a time reuses + * each loaded weight vector four times, and the outer panel loop keeps the + * activations that those four rows share resident in L1 while the weights + * stream past once. + * + * Both implementations are always compiled and chosen through this table, so + * one binary can compare them. + */ + +#ifndef PYSEG_KERNELS_H +#define PYSEG_KERNELS_H + +/* Output rows processed against the full weight matrix before moving on. Sized + * so the activation window and the output panel together stay inside L1. */ +#define PYSEG_MPANEL 32 + +typedef struct { + const char *name; + + /* For s in 0..m-1 and i in 0..n-1: + * y[s*ldy + i] = (bias ? bias[i] : 0) + dot(a + s*lda, w + i*k, k) + * k must be a multiple of four. */ + void (*gemm_nt)(float *y, + int ldy, + const float *a, + int lda, + const float *w, + const float *bias, + int m, + int n, + int k); + + /* One LSTM cell update in place. The preactivations are the sum of rec, + * the hidden-to-hidden product, and in, the projected input for this + * time step; both hold the four gate blocks in PyTorch's order, input then + * forget then candidate then output, each hid long: + * c = sigmoid(f) * c + sigmoid(i) * tanh(g) + * h = sigmoid(o) * tanh(c) + * + * This is here rather than in the forward pass because it is not a + * negligible epilogue. A window drives 4712 cell updates, which is three + * million logistic and hyperbolic evaluations; called one at a time + * through libm that costs more than either convolution stage. hid is a + * multiple of four. */ + void (*lstm_cell)(float *h, float *c, const float *rec, const float *in, int hid); +} pyseg_kernels; + +const pyseg_kernels *pyseg_kernels_scalar(void); + +/* The best set this build supports; equals the scalar set on architectures + * without an implementation here. */ +const pyseg_kernels *pyseg_kernels_simd(void); + +#endif /* PYSEG_KERNELS_H */ diff --git a/src/pyseg.c b/src/pyseg.c new file mode 100644 index 0000000..6860d54 --- /dev/null +++ b/src/pyseg.c @@ -0,0 +1,503 @@ +/* pyannote/segmentation-3.0 forward pass. + * + * Per window: normalise the waveform, one 251-tap filterbank and two ordinary + * convolutions each followed by max-pooling and per-channel normalisation, a + * four-layer bidirectional LSTM over the resulting 589 frames, two feed-forward + * layers and a seven-way softmax. Every buffer is allocated once in + * pyseg_create and reused, so a window costs no allocation, no syscall and no + * lock. The working set is 5.4 MB of activations plus 6 MB of read-only + * weights, which are shared by every engine mapping the same blob. + * + * Activations are stored time-major throughout, so the taps a convolution + * needs for one output step are one contiguous span and the whole network, + * convolutions included, runs through the single kernel in kernels.h. + * + * The convolutions are computed in tiles whose outputs are max-pooled on the + * spot. Only the pooled result is ever stored: the filterbank produces 15975 + * steps of 80 channels, 5.1 MB, of which only the 1.7 MB pooled form is needed + * downstream, and the same holds for the two later stages. */ + +#include "pyseg.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#include "kernels.h" +#include "weights.h" + +/* Stage geometry. Each convolution is followed by max-pooling by three, so + * only a multiple of three of its output steps is ever used; the last one or + * two steps of the later stages fall off the end and are not computed. */ +#define SINC_K 256 /* 251 taps padded with five zero weights */ +#define SINC_HOP 10 /* filterbank stride */ +#define C1 80 /* filterbank channels */ +#define T1 15975 /* filterbank output steps */ +#define P1 5325 /* after pooling */ + +#define C2 60 +#define K2 400 /* five taps of 80 channels */ +#define T2 5319 /* of 5321 produced by a valid kernel-5 convolution */ +#define P2 1773 + +#define C3 60 +#define K3 300 /* five taps of 60 channels */ +#define T3 1767 /* of 1769 */ +#define P3 PYSEG_FRAMES + +#define HID 128 +#define GATES (4 * HID) +#define BIDIR (2 * HID) + +/* Output steps per convolution tile. A multiple of three so pooling groups + * never straddle a tile boundary, and small enough that the tile stays in L1 + * while the weights stream past. */ +#define TILE 384 + +#define INSTANCE_NORM_EPS 1e-5 +#define LEAKY_SLOPE 0.01f + +/* Per-window buffer lengths, in floats. Everything before the LSTM is scratch + * shared by the windows of a batch, because they run through the front end one + * at a time; from the LSTM on, a batch is processed together and each window + * needs its own. */ +#define FEAT_STRIDE ((size_t)PYSEG_FRAMES * C3) +#define PRE_STRIDE ((size_t)PYSEG_FRAMES * GATES) +#define ACT_STRIDE ((size_t)PYSEG_FRAMES * BIDIR) + +struct pyseg { + const pyseg_kernels *k; + pyseg_weights w; + int ftz; + int batch; + + void *map; + size_t map_len; + size_t arena_bytes; + + void *arena; + float *wav; /* [PYSEG_WINDOW_SAMPLES + 8] */ + float *pool1; /* [P1][C1] */ + float *pool2; /* [P2][C2] */ + float *tile; /* [TILE][C1] */ + float *feat; /* [batch][P3][C3] */ + float *pre; /* [batch][PYSEG_FRAMES][GATES] */ + float *acta; /* [batch][PYSEG_FRAMES][BIDIR] */ + float *actb; /* [batch][PYSEG_FRAMES][BIDIR] */ + float *gates; /* [batch][GATES] */ + float *state; /* [2][batch][HID]: hidden then cell */ +}; + +/* ---- denormal handling ------------------------------------------------ + * + * The LSTM cell state decays toward zero across long silences, and denormal + * operands are slow on several targets. Flushing them is scoped to the call so + * the library never leaves the caller's floating point mode altered. */ + +typedef struct { + unsigned long long saved; + int active; +} fp_scope; + +#if defined(__aarch64__) +# define FPCR_FZ (1ull << 24) +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->active = enable; + if (!enable) + return; + unsigned long long r; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(r)); + s->saved = r; + if (!(r & FPCR_FZ)) + __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ)); +} +static inline void fp_scope_leave(const fp_scope *s) { + if (s->active) + __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved)); +} +#elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64) +# include <xmmintrin.h> +/* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring + * denormals-are-zero bit is left alone: it only affects denormal inputs, and + * it is absent on the earliest x86-64 parts, where writing it would fault. */ +# define MXCSR_FTZ 0x8000u +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->active = enable; + if (!enable) + return; + s->saved = _mm_getcsr(); + _mm_setcsr((unsigned)s->saved | MXCSR_FTZ); +} +static inline void fp_scope_leave(const fp_scope *s) { + if (s->active) + _mm_setcsr((unsigned)s->saved); +} +#else +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->saved = 0; + s->active = enable; +} +static inline void fp_scope_leave(const fp_scope *s) { (void)s; } +#endif + +/* ---- pointwise ------------------------------------------------------- */ + +static inline float leaky_(float x) { return x >= 0.0f ? x : LEAKY_SLOPE * x; } + +/* ---- normalisation ---------------------------------------------------- + * + * InstanceNorm1d over the whole window, one statistic pair per channel. The + * moments are accumulated in double and the deviations in a second pass: the + * filterbank output runs to five significant digits over 5325 steps, and a + * single-pass float variance loses the low ones to cancellation. + * + * This is also why the model cannot be run as a stream. Every frame's value + * depends on statistics of the entire ten-second window. */ +#define MAX_CHANNELS C1 + +static void instance_norm_leaky(float *x, int steps, int ch, const float *scale, const float *shift) { + double mean[MAX_CHANNELS], inv[MAX_CHANNELS]; + + for (int c = 0; c < ch; ++c) + mean[c] = 0.0; + for (int t = 0; t < steps; ++t) { + const float *row = x + (size_t)t * ch; + for (int c = 0; c < ch; ++c) + mean[c] += row[c]; + } + for (int c = 0; c < ch; ++c) + mean[c] /= steps; + + for (int c = 0; c < ch; ++c) + inv[c] = 0.0; + for (int t = 0; t < steps; ++t) { + const float *row = x + (size_t)t * ch; + for (int c = 0; c < ch; ++c) { + const double d = row[c] - mean[c]; + inv[c] += d * d; + } + } + for (int c = 0; c < ch; ++c) + inv[c] = 1.0 / sqrt(inv[c] / steps + INSTANCE_NORM_EPS); + + for (int t = 0; t < steps; ++t) { + float *row = x + (size_t)t * ch; + for (int c = 0; c < ch; ++c) + row[c] = leaky_((float)((row[c] - mean[c]) * inv[c] * scale[c] + shift[c])); + } +} + +/* ---- convolution stages ---------------------------------------------- + * + * One call covers a whole stage: convolution, optional magnitude, pooling by + * three, then the channel statistics. lda is how far apart consecutive + * output windows start in the source, which is the filterbank's hop for the + * first stage and one frame for the later ones. */ +static void conv_pool_norm(pyseg_t *v, + float *dst, + const float *src, + int lda, + const float *weight, + const float *bias, + int steps, + int ch, + int k, + int take_abs, + const float *scale, + const float *shift) { + for (int t0 = 0; t0 < steps; t0 += TILE) { + const int n = steps - t0 < TILE ? steps - t0 : TILE; + v->k->gemm_nt(v->tile, ch, src + (size_t)t0 * lda, lda, weight, bias, n, ch, k); + + float *out = dst + (size_t)(t0 / 3) * ch; + for (int p = 0; p < n / 3; ++p) { + const float *a = v->tile + (size_t)(3 * p) * ch; + const float *b = a + ch, *c = b + ch; + float *o = out + (size_t)p * ch; + if (take_abs) { + for (int i = 0; i < ch; ++i) { + const float x = fabsf(a[i]), y = fabsf(b[i]), z = fabsf(c[i]); + o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z); + } + } else { + for (int i = 0; i < ch; ++i) { + const float x = a[i], y = b[i], z = c[i]; + o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z); + } + } + } + } + instance_norm_leaky(dst, steps / 3, ch, scale, shift); +} + +/* ---- LSTM ------------------------------------------------------------- + * + * The input projection of every time step shares one weight matrix, so it runs + * as a single pass over every step of every window in the batch before the + * recurrence starts, which is three fifths of the layer's arithmetic done at + * full matrix-multiply efficiency. + * + * What is left is the hidden-to-hidden product, one per step, and it is the + * reason batching exists. On its own it reads a 256 KB weight matrix to + * perform 65536 multiply-adds, so it runs at the speed of memory rather than + * of arithmetic and is the single most expensive part of a window. Windows are + * independent, so running several of them in step turns that matrix-vector + * product into a small matrix-matrix product: one weight load then serves + * every window, and the recurrence speeds up by better than a factor of two at + * a batch of four. */ +static void lstm_direction(pyseg_t *v, + int count, + const float *in, + int lda, + int k, + const pyseg_lstm_weights *w, + float *out, + int offset, + int reverse) { + v->k->gemm_nt(v->pre, GATES, in, lda, w->ih, w->bias, count * PYSEG_FRAMES, GATES, k); + + float *h = v->state, *c = v->state + (size_t)v->batch * HID; + memset(h, 0, (size_t)count * HID * sizeof(float)); + memset(c, 0, (size_t)count * HID * sizeof(float)); + + for (int step = 0; step < PYSEG_FRAMES; ++step) { + const int t = reverse ? PYSEG_FRAMES - 1 - step : step; + v->k->gemm_nt(v->gates, GATES, h, HID, w->hh, NULL, count, GATES, HID); + for (int b = 0; b < count; ++b) { + float *hb = h + (size_t)b * HID; + v->k->lstm_cell(hb, c + (size_t)b * HID, v->gates + (size_t)b * GATES, + v->pre + (size_t)b * PRE_STRIDE + (size_t)t * GATES, HID); + memcpy(out + (size_t)b * ACT_STRIDE + (size_t)t * BIDIR + offset, hb, + HID * sizeof(float)); + } + } +} + +/* ---- forward pass ---------------------------------------------------- */ + +/* Everything up to the LSTM, for one window of the batch. */ +static void front_end(pyseg_t *v, int slot) { + const pyseg_weights *w = &v->w; + + /* Waveform normalisation: one channel, statistics over the whole window. */ + { + double mean = 0.0, var = 0.0; + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) + mean += v->wav[i]; + mean /= PYSEG_WINDOW_SAMPLES; + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) { + const double d = v->wav[i] - mean; + var += d * d; + } + const double inv = 1.0 / sqrt(var / PYSEG_WINDOW_SAMPLES + INSTANCE_NORM_EPS); + const float scale = w->wavnorm[0], shift = w->wavnorm[1]; + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) + v->wav[i] = (float)((v->wav[i] - mean) * inv) * scale + shift; + } + + conv_pool_norm(v, v->pool1, v->wav, SINC_HOP, w->sinc_w, NULL, T1, C1, SINC_K, 1, w->norm_w[0], + w->norm_b[0]); + conv_pool_norm(v, v->pool2, v->pool1, C1, w->conv_w[0], w->conv_b[0], T2, C2, K2, 0, w->norm_w[1], + w->norm_b[1]); + conv_pool_norm(v, v->feat + (size_t)slot * FEAT_STRIDE, v->pool2, C2, w->conv_w[1], w->conv_b[1], + T3, C3, K3, 0, w->norm_w[2], w->norm_b[2]); +} + +/* The LSTM stack, the feed-forward layers and the softmax, over a whole batch + * whose features the front end has already produced. */ +static void back_end(pyseg_t *v, int count, float *probs) { + const pyseg_weights *w = &v->w; + const pyseg_kernels *k = v->k; + const int rows = count * PYSEG_FRAMES; + + lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][0], v->acta, 0, 0); + lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][1], v->acta, HID, 1); + for (int layer = 1; layer < 4; ++layer) { + const float *in = (layer & 1) ? v->acta : v->actb; + float *out = (layer & 1) ? v->actb : v->acta; + lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][0], out, 0, 0); + lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][1], out, HID, 1); + } + + /* Layer 3 wrote actb; the feed-forward stack then ping-pongs back. Both + * layers are narrower than what they read, so writing over the buffer the + * previous layer occupied is safe. */ + k->gemm_nt(v->acta, HID, v->actb, BIDIR, w->lin_w[0], w->lin_b[0], rows, HID, BIDIR); + for (int i = 0; i < rows * HID; ++i) + v->acta[i] = leaky_(v->acta[i]); + k->gemm_nt(v->actb, HID, v->acta, HID, w->lin_w[1], w->lin_b[1], rows, HID, HID); + for (int i = 0; i < rows * HID; ++i) + v->actb[i] = leaky_(v->actb[i]); + + k->gemm_nt(probs, PYSEG_CLASSES, v->actb, HID, w->cls_w, w->cls_b, rows, PYSEG_CLASSES, HID); + + for (int t = 0; t < rows; ++t) { + float *row = probs + (size_t)t * PYSEG_CLASSES; + float max = row[0]; + for (int i = 1; i < PYSEG_CLASSES; ++i) + if (row[i] > max) + max = row[i]; + float sum = 0.0f; + for (int i = 0; i < PYSEG_CLASSES; ++i) { + row[i] = expf(row[i] - max); + sum += row[i]; + } + const float inv = 1.0f / sum; + for (int i = 0; i < PYSEG_CLASSES; ++i) + row[i] *= inv; + } +} + +/* ---- lifecycle ------------------------------------------------------- */ + +pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch) { + if (batch < 1 || batch > PYSEG_MAX_BATCH) + return NULL; + + pyseg_t *v = calloc(1, sizeof *v); + if (!v) + return NULL; + v->batch = batch; + + if (path) { + const int verify = !(flags & PYSEG_NO_VERIFY); + if (pyseg_weights_map(path, verify, &v->w, &v->map, &v->map_len) != 0) { + free(v); + return NULL; + } + } else { + size_t len = 0; + const void *blob = pyseg_weights_embedded(&len); + if (!blob || pyseg_weights_parse(blob, len, 0, &v->w) != 0) { + free(v); + return NULL; + } + } + + /* One allocation, carved into 64-byte aligned buffers in this order. */ + const size_t b = (size_t)batch; + const size_t sizes[] = { + PYSEG_WINDOW_SAMPLES + 8, /* wav, with slack past the last filter window */ + (size_t)P1 * C1, /* pool1 */ + (size_t)P2 * C2, /* pool2 */ + (size_t)TILE * C1, /* tile */ + b * FEAT_STRIDE, /* feat */ + b * PRE_STRIDE, /* pre */ + b * ACT_STRIDE, /* acta */ + b * ACT_STRIDE, /* actb */ + b * GATES, /* gates */ + 2 * b * HID, /* state */ + }; + float **slots[] = {&v->wav, &v->pool1, &v->pool2, &v->tile, &v->feat, + &v->pre, &v->acta, &v->actb, &v->gates, &v->state}; + + size_t total = 0; + for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) + total += (sizes[i] * sizeof(float) + 63) / 64 * 64; + if (posix_memalign(&v->arena, 64, total) != 0 || !v->arena) { + pyseg_weights_unmap(v->map, v->map_len); + free(v); + return NULL; + } + memset(v->arena, 0, total); + v->arena_bytes = total; + + unsigned char *p = v->arena; + for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) { + *slots[i] = (float *)(void *)p; + p += (sizes[i] * sizeof(float) + 63) / 64 * 64; + } + + v->k = (flags & PYSEG_SCALAR_KERNELS) ? pyseg_kernels_scalar() : pyseg_kernels_simd(); + v->ftz = (flags & PYSEG_NO_FTZ) == 0; + return v; +} + +pyseg_t *pyseg_create_ex(const char *path, unsigned flags) { + return pyseg_create_batch(path, flags, 1); +} + +pyseg_t *pyseg_create(void) { return pyseg_create_ex(NULL, 0); } + +pyseg_t *pyseg_create_from_file(const char *path) { return pyseg_create_ex(path, 0); } + +void pyseg_destroy(pyseg_t *v) { + if (!v) + return; + pyseg_weights_unmap(v->map, v->map_len); + free(v->arena); + free(v); +} + +const char *pyseg_backend_name(const pyseg_t *v) { return v->k->name; } + +int pyseg_batch(const pyseg_t *v) { return v->batch; } + +size_t pyseg_memory_bytes(const pyseg_t *v) { + return sizeof *v + v->arena_bytes + (v->map ? v->map_len : 0); +} + +/* ---- entry points ----------------------------------------------------- */ + +void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs) { + if (count < 1 || count > v->batch) + return; + fp_scope fp; + fp_scope_enter(&fp, v->ftz); + for (int b = 0; b < count; ++b) { + memcpy(v->wav, samples + (size_t)b * PYSEG_WINDOW_SAMPLES, + PYSEG_WINDOW_SAMPLES * sizeof(float)); + front_end(v, b); + } + back_end(v, count, probs); + fp_scope_leave(&fp); +} + +void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs) { + pyseg_process_batch_f32(v, samples, 1, probs); +} + +void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs) { + fp_scope fp; + fp_scope_enter(&fp, v->ftz); + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) + v->wav[i] = (float)samples[i] * (1.0f / 32768.0f); + front_end(v, 0); + back_end(v, 1, probs); + fp_scope_leave(&fp); +} + +/* ---- powerset decoding ----------------------------------------------- + * + * The seven classes are, in order, the empty set, the three singletons and the + * three pairs, which is the order itertools.combinations produces and the + * order the classifier was trained in. */ +static const unsigned char powerset_members[PYSEG_CLASSES] = { + 0x0, 0x1, 0x2, 0x4, 0x3, 0x5, 0x6, +}; + +void pyseg_to_multilabel(const float *probs, float *active) { + for (int t = 0; t < PYSEG_FRAMES; ++t) { + const float *row = probs + (size_t)t * PYSEG_CLASSES; + int best = 0; + for (int i = 1; i < PYSEG_CLASSES; ++i) + if (row[i] > row[best]) + best = i; + const unsigned mask = powerset_members[best]; + float *out = active + (size_t)t * PYSEG_SPEAKERS; + for (int s = 0; s < PYSEG_SPEAKERS; ++s) + out[s] = (mask >> s) & 1u ? 1.0f : 0.0f; + } +} + +int pyseg_powerset_class(int speaker_a, int speaker_b) { + unsigned mask = 0; + if (speaker_a >= 0) + mask |= 1u << speaker_a; + if (speaker_b >= 0) + mask |= 1u << speaker_b; + for (int i = 0; i < PYSEG_CLASSES; ++i) + if (powerset_members[i] == mask) + return i; + return -1; +} diff --git a/src/pyseg.h b/src/pyseg.h new file mode 100644 index 0000000..b08f7b3 --- /dev/null +++ b/src/pyseg.h @@ -0,0 +1,215 @@ +/* pyseg: pyannote/segmentation-3.0 speaker segmentation in portable C. + * + * The model reads exactly ten seconds of 16 kHz mono audio and returns, for + * each of 589 output frames, a distribution over seven classes: no speaker, + * one of three speakers alone, or one of the three pairs. That "powerset" + * encoding is what the model was trained on; pyseg_to_multilabel turns it into + * per-speaker activity, and the sliding-window driver in this header turns a + * recording of any length into speech, overlap and speaker-count tracks. + * + * Speaker identities hold only inside a single window. Two windows that + * overlap may number the same person differently, so joining windows into + * whole-recording speakers needs a speaker embedding model, which is outside + * the scope of this library. Speech activity, overlapped speech and the + * per-frame speaker count do carry across windows and are what the sliding + * window aggregates. + * + * One engine handles one window at a time and owns all its scratch, so a + * process runs as many engines as it has threads. No allocation, no syscall + * and no lock happens inside pyseg_process. */ + +#ifndef PYSEG_H +#define PYSEG_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define PYSEG_VERSION_MAJOR 1 +#define PYSEG_VERSION_MINOR 0 + +#define PYSEG_SAMPLE_RATE 16000 +#define PYSEG_WINDOW_SAMPLES 160000 /* 10 s */ +#define PYSEG_FRAMES 589 +#define PYSEG_CLASSES 7 /* powerset classes */ +#define PYSEG_SPEAKERS 3 + +/* Output frame grid, in samples: frame f covers the 991 input samples starting + * at f * PYSEG_FRAME_STEP, relative to the start of the window. */ +#define PYSEG_FRAME_STEP 270 +#define PYSEG_FRAME_SIZE 991 + +/* ---- engine ---------------------------------------------------------- */ + +typedef struct pyseg pyseg_t; + +/* Windows an engine can hold at once, and engines pyseg_slide will run at + * once. Both are bounded so a caller cannot turn a large number into an + * allocation proportional to it: each engine holds several megabytes. */ +#define PYSEG_MAX_BATCH 64 +#define PYSEG_MAX_THREADS 256 + +/* Creation flags. The defaults (0) select the fastest kernels available. */ +enum { + PYSEG_SCALAR_KERNELS = 1u << 0, /* portable C kernels, no SIMD intrinsics */ + PYSEG_NO_FTZ = 1u << 1, /* leave denormal handling to the caller */ + PYSEG_NO_VERIFY = 1u << 2, /* skip checksum verification of a weight file */ +}; + +/* Uses the weights compiled into the binary. Returns NULL when the build has + * none, or on allocation failure. */ +pyseg_t *pyseg_create(void); + +/* Reads a blob produced by tools/export_weights.py. The mapping is private to + * the engine and released by pyseg_destroy. */ +pyseg_t *pyseg_create_from_file(const char *path); + +pyseg_t *pyseg_create_ex(const char *path, unsigned flags); + +/* An engine that processes batch windows per call. + * + * Batching exists for one reason: the LSTM recurrence reads a 256 KB weight + * matrix per time step to do 65536 multiply-adds, so on its own it runs at the + * speed of memory. Windows are independent, so several of them stepped + * together share each weight load, and a batch of four cuts the recurrence, + * which is nearly half of a window's cost, by better than half. + * + * The price is memory: the front end's scratch is shared, but each window + * needs about 2.5 MB of its own from the LSTM on. A batch of one is the + * smallest footprint and the lowest latency for a single window; four is the + * best throughput per byte on the machines measured. Batches are also the + * wrong lever for using more cores; for that, run one engine per thread. + * + * Returns NULL for a batch outside 1..PYSEG_MAX_BATCH. */ +pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch); + +int pyseg_batch(const pyseg_t *v); + +void pyseg_destroy(pyseg_t *v); + +/* Name of the active kernel set, for logs and benchmark output. */ +const char *pyseg_backend_name(const pyseg_t *v); + +/* Bytes of scratch and weights the engine holds, for capacity planning. */ +size_t pyseg_memory_bytes(const pyseg_t *v); + +/* One window. samples holds PYSEG_WINDOW_SAMPLES values; the f32 variant + * expects [-1, 1] and the s16 variant is scaled by 1/32768 to match. probs + * receives PYSEG_FRAMES * PYSEG_CLASSES values, frame-major, each frame a + * distribution summing to one. */ +void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs); +void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs); + +/* count consecutive windows, at most pyseg_batch(); outside that range the + * call does nothing and leaves probs untouched. samples holds + * count * PYSEG_WINDOW_SAMPLES values and probs receives + * count * PYSEG_FRAMES * PYSEG_CLASSES, both window-major. Results do not + * depend on how windows are grouped into batches, only on the windows + * themselves. */ +void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs); + +/* ---- powerset decoding ----------------------------------------------- */ + +/* Per-frame speaker activity from powerset probabilities: active receives + * PYSEG_FRAMES * PYSEG_SPEAKERS values, 1.0 where the most likely class + * contains that speaker and 0.0 elsewhere. This is the hard decoding pyannote + * uses; it is not a per-speaker probability. */ +void pyseg_to_multilabel(const float *probs, float *active); + +/* Class index whose members are exactly the given speaker set, for tests. */ +int pyseg_powerset_class(int speaker_a, int speaker_b); + +/* ---- sliding window over a whole recording --------------------------- */ + +/* Tracks produced per aggregated frame, in this order. */ +enum { + PYSEG_TRACK_SPEECH = 0, /* at least one speaker */ + PYSEG_TRACK_OVERLAP = 1, /* at least two speakers */ + PYSEG_TRACK_COUNT = 2, /* expected number of speakers */ + PYSEG_TRACKS = 3, +}; + +typedef struct { + /* Seconds between consecutive windows. pyannote's default is 1.0, one + * tenth of the window, which is also what its pipelines are tuned for. + * Larger steps cost proportionally less and blur the aggregate. */ + double step; + /* Threads used for the window loop. 0 or 1 runs on the calling thread; + * more than PYSEG_MAX_THREADS is clamped to it. */ + int threads; + /* Windows each thread's engine processes per call; see + * pyseg_create_batch. 0 selects the default. */ + int batch; + /* Passed to pyseg_create_ex for the per-thread engines. */ + unsigned flags; + /* Weight file for those engines; NULL uses the embedded blob. */ + const char *weights; +} pyseg_slide_params; + +void pyseg_slide_params_default(pyseg_slide_params *p); + +/* Number of aggregated frames a recording of num_samples produces. */ +size_t pyseg_slide_frames(size_t num_samples, double step); + +/* Runs the model over the whole recording and overlap-adds the windows the way + * pyannote's Inference does: a Hamming taper across each window's frames, then + * a weighted average per frame. tracks receives + * pyseg_slide_frames() * PYSEG_TRACKS values, frame-major. + * + * Returns the number of aggregated frames, or 0 on failure. */ +size_t pyseg_slide(const float *samples, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks); + +/* The same over 16-bit samples, scaled by 1/32768 one window at a time. + * Recordings arrive as 16-bit PCM, and holding an hour of it as float32 costs + * 230 MB against 115 MB, more than everything else this library allocates put + * together. */ +size_t pyseg_slide_s16(const int16_t *samples, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks); + +/* Seconds covered by aggregated frame f. The grid matches the model's own + * receptive field, so frame centres are PYSEG_FRAME_STEP samples apart. */ +double pyseg_frame_time(size_t frame); + +/* ---- binarization ---------------------------------------------------- */ + +typedef struct { + double start; + double end; +} pyseg_segment; + +typedef struct { + double onset; /* enter a region above this score */ + double offset; /* leave it below this one */ + double min_duration_on; /* drop regions shorter than this, in seconds */ + double min_duration_off; /* fill gaps shorter than this, in seconds */ +} pyseg_binarize_params; + +void pyseg_binarize_params_default(pyseg_binarize_params *p); + +/* Turns one track of an aggregated score sequence into segments, mirroring + * pyannote.audio.utils.signal.Binarize. scores is strided by PYSEG_TRACKS. + * + * Returns the number of segments found. Nothing is written unless the whole + * list fits, because the gap-filling pass is defined over the complete list; + * when out is NULL or too small, size a buffer from the return value and + * call again. */ +size_t pyseg_binarize(const float *scores, + size_t num_frames, + int track, + const pyseg_binarize_params *params, + pyseg_segment *out, + size_t out_cap); + +#ifdef __cplusplus +} +#endif + +#endif /* PYSEG_H */ diff --git a/src/pyseg_cli.c b/src/pyseg_cli.c new file mode 100644 index 0000000..f694f57 --- /dev/null +++ b/src/pyseg_cli.c @@ -0,0 +1,176 @@ +/* Command line front end: a 16 kHz mono WAV in, segments or frame scores out. */ + +#include "pyseg.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "wav.h" + +static const char *usage = "usage: pyseg_cli [options] file.wav\n" + "\n" + " --mode speech|overlap|count|frames what to print (default speech)\n" + " --step SECONDS window hop, default 1.0 (a tenth of the window)\n" + " --threads N windows to run in parallel, default 1\n" + " --batch N windows per engine call, default 4\n" + " --weights FILE weight blob; default is the embedded copy\n" + " --onset X score above which a region opens, default 0.5\n" + " --offset X score below which it closes, default 0.5\n" + " --min-on SECONDS drop regions shorter than this, default 0\n" + " --min-off SECONDS fill gaps shorter than this, default 0\n" + " --scalar portable C kernels instead of SIMD\n" + "\n" + "speech and overlap print [start end] pairs in seconds; count and frames\n" + "print one line per frame.\n"; + +enum { MODE_SPEECH, MODE_OVERLAP, MODE_COUNT, MODE_FRAMES }; + +static int parse_mode(const char *s, int *mode) { + if (strcmp(s, "speech") == 0 || strcmp(s, "vad") == 0) + *mode = MODE_SPEECH; + else if (strcmp(s, "overlap") == 0 || strcmp(s, "osd") == 0) + *mode = MODE_OVERLAP; + else if (strcmp(s, "count") == 0) + *mode = MODE_COUNT; + else if (strcmp(s, "frames") == 0) + *mode = MODE_FRAMES; + else + return -1; + return 0; +} + +int main(int argc, char **argv) { + pyseg_slide_params slide; + pyseg_binarize_params bin; + pyseg_slide_params_default(&slide); + pyseg_binarize_params_default(&bin); + + int mode = MODE_SPEECH; + const char *path = NULL; + + for (int i = 1; i < argc; ++i) { + const char *a = argv[i]; + const char *v = (i + 1 < argc) ? argv[i + 1] : NULL; +#define NEED(name) \ + if (!v) { \ + fprintf(stderr, "%s needs a value\n", name); \ + return 2; \ + } + if (strcmp(a, "--mode") == 0) { + NEED("--mode"); + if (parse_mode(v, &mode) != 0) { + fprintf(stderr, "unknown mode: %s\n", v); + return 2; + } + ++i; + } else if (strcmp(a, "--step") == 0) { + NEED("--step"); + slide.step = atof(v); + ++i; + } else if (strcmp(a, "--threads") == 0) { + NEED("--threads"); + slide.threads = atoi(v); + ++i; + } else if (strcmp(a, "--batch") == 0) { + NEED("--batch"); + slide.batch = atoi(v); + ++i; + } else if (strcmp(a, "--weights") == 0) { + NEED("--weights"); + slide.weights = v; + ++i; + } else if (strcmp(a, "--onset") == 0) { + NEED("--onset"); + bin.onset = atof(v); + ++i; + } else if (strcmp(a, "--offset") == 0) { + NEED("--offset"); + bin.offset = atof(v); + ++i; + } else if (strcmp(a, "--min-on") == 0) { + NEED("--min-on"); + bin.min_duration_on = atof(v); + ++i; + } else if (strcmp(a, "--min-off") == 0) { + NEED("--min-off"); + bin.min_duration_off = atof(v); + ++i; + } else if (strcmp(a, "--scalar") == 0) { + slide.flags |= PYSEG_SCALAR_KERNELS; + } else if (strcmp(a, "-h") == 0 || strcmp(a, "--help") == 0) { + fputs(usage, stdout); + return 0; + } else if (a[0] == '-') { + fprintf(stderr, "unknown option: %s\n%s", a, usage); + return 2; + } else { + path = a; + } +#undef NEED + } + + if (!path) { + fputs(usage, stderr); + return 2; + } + + wav_t wav; + const char *err = NULL; + if (wav_read_mono_s16(path, &wav, &err) != 0) { + fprintf(stderr, "%s: %s\n", path, err ? err : "read failed"); + return 1; + } + if (wav.sample_rate != PYSEG_SAMPLE_RATE) { + fprintf(stderr, "%s: expected %d Hz, got %d\n", path, PYSEG_SAMPLE_RATE, wav.sample_rate); + wav_free(&wav); + return 1; + } + + const size_t count = wav.count; + const size_t frames = pyseg_slide_frames(count, slide.step); + float *tracks = malloc(frames * PYSEG_TRACKS * sizeof(float)); + if (!tracks) { + wav_free(&wav); + return 1; + } + + const size_t got = pyseg_slide_s16(wav.samples, count, &slide, tracks); + wav_free(&wav); + if (got == 0) { + fprintf(stderr, "inference failed: no engine could be created%s\n", + slide.weights ? " from the given weight file" + : "; this build may carry no embedded weights, " + "in which case pass --weights pyseg_weights.bin"); + free(tracks); + return 1; + } + + if (mode == MODE_FRAMES || mode == MODE_COUNT) { + for (size_t f = 0; f < got; ++f) { + const float *row = tracks + f * PYSEG_TRACKS; + if (mode == MODE_COUNT) + printf("%.3f %.4f\n", pyseg_frame_time(f), row[PYSEG_TRACK_COUNT]); + else + printf("%.3f %.4f %.4f %.4f\n", pyseg_frame_time(f), row[PYSEG_TRACK_SPEECH], + row[PYSEG_TRACK_OVERLAP], row[PYSEG_TRACK_COUNT]); + } + free(tracks); + return 0; + } + + const int track = mode == MODE_SPEECH ? PYSEG_TRACK_SPEECH : PYSEG_TRACK_OVERLAP; + const size_t n = pyseg_binarize(tracks, got, track, &bin, NULL, 0); + pyseg_segment *segments = n ? malloc(n * sizeof *segments) : NULL; + if (n && !segments) { + free(tracks); + return 1; + } + const size_t written = pyseg_binarize(tracks, got, track, &bin, segments, n); + for (size_t i = 0; i < written; ++i) + printf("%.3f %.3f\n", segments[i].start, segments[i].end); + + free(segments); + free(tracks); + return 0; +} diff --git a/src/slide.c b/src/slide.c new file mode 100644 index 0000000..be23ec4 --- /dev/null +++ b/src/slide.c @@ -0,0 +1,423 @@ +/* Sliding-window inference over a whole recording, and binarization. + * + * The model sees ten seconds at a time and its per-frame statistics depend on + * the whole window, so a recording is covered by overlapping windows and their + * frame scores are averaged. pyannote weights that average with a Hamming + * taper, which lets a frame near a window's edge, where the LSTM has seen + * context on one side only, defer to the window that holds it in the middle. + * + * Windows are independent, so the loop parallelises exactly. Each thread takes + * a contiguous run of windows and accumulates into a private plane covering + * only the frames that run reaches; the planes of two adjacent threads overlap + * by at most one window's worth of frames, and the main thread sums them at + * the end. Contiguous runs rather than interleaved ones are what keeps those + * planes small: interleaving would give every thread a plane as large as the + * whole recording. */ + +#include "pyseg.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#ifdef PYSEG_THREADS +# include <pthread.h> +#endif + +#define AGG_EPSILON 1e-12f + +/* Four windows per engine call: the point at which sharing each LSTM weight + * load stops paying for the registers it costs, measured on both an Apple + * M-series core and an x86 server core. */ +#define DEFAULT_BATCH 4 + +void pyseg_slide_params_default(pyseg_slide_params *p) { + p->step = 1.0; + p->threads = 1; + p->batch = DEFAULT_BATCH; + p->flags = 0; + p->weights = NULL; +} + +void pyseg_binarize_params_default(pyseg_binarize_params *p) { + /* The powerset model emits hard per-frame decisions, so an aggregated + * score is the fraction of windows that voted yes and the thresholds sit + * at one half. pyannote fixes them there for this model rather than + * tuning them per corpus. */ + p->onset = 0.5; + p->offset = 0.5; + p->min_duration_on = 0.0; + p->min_duration_off = 0.0; +} + +double pyseg_frame_time(size_t frame) { + return (double)(frame * PYSEG_FRAME_STEP) / PYSEG_SAMPLE_RATE; +} + +/* Centre of a frame, which is where its score is defined and therefore where + * segment boundaries land. */ +static double frame_middle(size_t frame) { + return pyseg_frame_time(frame) + 0.5 * PYSEG_FRAME_SIZE / PYSEG_SAMPLE_RATE; +} + +/* Rounds n/d to the nearest integer, halves to even, in exact integer + * arithmetic. Window starts do not fall on frame boundaries, so which way a + * tie goes decides a frame of alignment against the reference. */ +static size_t rint_div(size_t n, size_t d) { + const size_t q = n / d, r = n % d; + if (2 * r < d) + return q; + if (2 * r > d) + return q + 1; + return q + (q & 1); +} + +/* Windows the recording is cut into: the complete ones plus, when the tail + * does not land on a window boundary, one zero-padded window covering it. */ +static size_t window_count(size_t num_samples, size_t step_samples) { + if (num_samples < PYSEG_WINDOW_SAMPLES) + return 1; + const size_t whole = (num_samples - PYSEG_WINDOW_SAMPLES) / step_samples + 1; + return whole + ((num_samples - PYSEG_WINDOW_SAMPLES) % step_samples > 0 ? 1 : 0); +} + +static size_t step_samples_of(double step) { + const double s = step * PYSEG_SAMPLE_RATE + 0.5; + return s < 1.0 ? 1 : (size_t)s; +} + +/* First aggregated frame a window starting at sample contributes to. */ +static size_t start_frame_of(size_t sample) { return rint_div(sample, PYSEG_FRAME_STEP); } + +size_t pyseg_slide_frames(size_t num_samples, double step) { + if (num_samples == 0 || step <= 0.0) + return 0; + const size_t step_samples = step_samples_of(step); + const size_t windows = window_count(num_samples, step_samples); + size_t frames = + rint_div(PYSEG_WINDOW_SAMPLES + (windows - 1) * step_samples, PYSEG_FRAME_STEP) + 1; + + /* A zero-padded tail window predicts past the end of the recording; those + * frames are dropped, as pyannote does after aggregating. */ + const int padded = + num_samples < PYSEG_WINDOW_SAMPLES || (num_samples - PYSEG_WINDOW_SAMPLES) % step_samples > 0; + if (padded) { + const size_t limit = num_samples / PYSEG_FRAME_STEP + 1; + if (limit < frames) + frames = limit; + } + return frames; +} + +/* ---- window loop ------------------------------------------------------ */ + +typedef struct { + const float *samples; /* one of these two is set */ + const int16_t *samples_s16; + size_t num_samples; + size_t step_samples; + size_t frames; + const float *taper; /* [PYSEG_FRAMES] */ + const char *weights; + unsigned flags; + int batch; +} slide_job; + +typedef struct { + const slide_job *job; + size_t first_window, last_window; /* half-open */ + size_t first_frame, frame_count; + float *sum; /* [frame_count][PYSEG_TRACKS] */ + float *count; /* [frame_count][PYSEG_TRACKS] */ + int failed; +} slide_worker; + +/* Copies one window out of the recording, zero-filling past the end. */ +static void gather(const slide_job *job, size_t index, float *dst) { + const size_t start = index * job->step_samples; + size_t n = 0; + if (start < job->num_samples) { + n = job->num_samples - start; + if (n > PYSEG_WINDOW_SAMPLES) + n = PYSEG_WINDOW_SAMPLES; + if (job->samples) + memcpy(dst, job->samples + start, n * sizeof(float)); + else + for (size_t i = 0; i < n; ++i) + dst[i] = (float)job->samples_s16[start + i] * (1.0f / 32768.0f); + } + memset(dst + n, 0, (PYSEG_WINDOW_SAMPLES - n) * sizeof(float)); +} + +/* Adds one window's decoded votes into this thread's plane. */ +static void accumulate(slide_worker *self, size_t index, const float *active) { + const slide_job *job = self->job; + const size_t base = start_frame_of(index * job->step_samples) - self->first_frame; + for (int t = 0; t < PYSEG_FRAMES; ++t) { + const size_t row = base + (size_t)t; + if (row >= self->frame_count) + break; + const float *a = active + (size_t)t * PYSEG_SPEAKERS; + const float speakers = a[0] + a[1] + a[2]; + const float weight = job->taper[t]; + float *sum = self->sum + row * PYSEG_TRACKS; + float *count = self->count + row * PYSEG_TRACKS; + sum[PYSEG_TRACK_SPEECH] += (speakers >= 1.0f ? weight : 0.0f); + sum[PYSEG_TRACK_OVERLAP] += (speakers >= 2.0f ? weight : 0.0f); + sum[PYSEG_TRACK_COUNT] += speakers * weight; + count[PYSEG_TRACK_SPEECH] += weight; + count[PYSEG_TRACK_OVERLAP] += weight; + count[PYSEG_TRACK_COUNT] += weight; + } +} + +static void *worker_run(void *arg) { + slide_worker *self = arg; + const slide_job *job = self->job; + const int batch = job->batch; + + pyseg_t *engine = pyseg_create_batch(job->weights, job->flags, batch); + float *windows = malloc((size_t)batch * PYSEG_WINDOW_SAMPLES * sizeof(float)); + float *probs = malloc((size_t)batch * PYSEG_FRAMES * PYSEG_CLASSES * sizeof(float)); + float *active = malloc((size_t)PYSEG_FRAMES * PYSEG_SPEAKERS * sizeof(float)); + if (!engine || !windows || !probs || !active) { + self->failed = 1; + goto done; + } + + for (size_t index = self->first_window; index < self->last_window;) { + int count = 0; + while (count < batch && index + (size_t)count < self->last_window) { + gather(job, index + (size_t)count, windows + (size_t)count * PYSEG_WINDOW_SAMPLES); + ++count; + } + pyseg_process_batch_f32(engine, windows, count, probs); + for (int b = 0; b < count; ++b) { + pyseg_to_multilabel(probs + (size_t)b * PYSEG_FRAMES * PYSEG_CLASSES, active); + accumulate(self, index + (size_t)b, active); + } + index += (size_t)count; + } + +done: + free(active); + free(probs); + free(windows); + pyseg_destroy(engine); + return NULL; +} + +static size_t slide_run(const float *samples, + const int16_t *samples_s16, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks) { + pyseg_slide_params defaults; + if (!params) { + pyseg_slide_params_default(&defaults); + params = &defaults; + } + if ((!samples && !samples_s16) || num_samples == 0 || !tracks || params->step <= 0.0) + return 0; + + const size_t step_samples = step_samples_of(params->step); + if (step_samples > PYSEG_WINDOW_SAMPLES || params->batch > PYSEG_MAX_BATCH) + return 0; + + const size_t windows = window_count(num_samples, step_samples); + const size_t frames = pyseg_slide_frames(num_samples, params->step); + if (frames == 0) + return 0; + + /* np.hamming over the window's frames. It never reaches zero, so every + * frame keeps a vote however close to the edge it sits. */ + float taper[PYSEG_FRAMES]; + for (int t = 0; t < PYSEG_FRAMES; ++t) + taper[t] = (float)(0.54 - 0.46 * cos(2.0 * M_PI * t / (PYSEG_FRAMES - 1))); + + const slide_job job = { + .samples = samples, + .samples_s16 = samples_s16, + .num_samples = num_samples, + .step_samples = step_samples, + .frames = frames, + .taper = taper, + .weights = params->weights, + .flags = params->flags, + .batch = params->batch > 0 ? params->batch : DEFAULT_BATCH, + }; + + int nthreads = params->threads > 0 ? params->threads : 1; + if (nthreads > PYSEG_MAX_THREADS) + nthreads = PYSEG_MAX_THREADS; +#ifndef PYSEG_THREADS + nthreads = 1; +#endif + if ((size_t)nthreads > windows) + nthreads = (int)windows; + + slide_worker *workers = calloc((size_t)nthreads, sizeof *workers); + if (!workers) + return 0; + + int failed = 0; + for (int i = 0; i < nthreads; ++i) { + slide_worker *w = &workers[i]; + w->job = &job; + w->first_window = windows * (size_t)i / (size_t)nthreads; + w->last_window = windows * (size_t)(i + 1) / (size_t)nthreads; + w->first_frame = start_frame_of(w->first_window * step_samples); + size_t end = start_frame_of((w->last_window - 1) * step_samples) + PYSEG_FRAMES; + if (end > frames) + end = frames; + w->frame_count = end > w->first_frame ? end - w->first_frame : 0; + w->sum = calloc(w->frame_count * PYSEG_TRACKS + 1, sizeof(float)); + w->count = calloc(w->frame_count * PYSEG_TRACKS + 1, sizeof(float)); + if (!w->sum || !w->count) + failed = 1; + } + +#ifdef PYSEG_THREADS + pthread_t *threads = nthreads > 1 ? calloc((size_t)nthreads - 1, sizeof *threads) : NULL; + int spawned = 0; + if (!failed && nthreads > 1 && !threads) + failed = 1; + if (!failed) + for (int i = 1; i < nthreads; ++i) { + if (pthread_create(&threads[spawned], NULL, worker_run, &workers[i]) != 0) { + failed = 1; + break; + } + ++spawned; + } +#endif + if (!failed) + worker_run(&workers[0]); +#ifdef PYSEG_THREADS + for (int i = 0; i < spawned; ++i) + pthread_join(threads[i], NULL); + free(threads); +#endif + + for (int i = 0; i < nthreads; ++i) + failed |= workers[i].failed; + + if (!failed) { + memset(tracks, 0, frames * PYSEG_TRACKS * sizeof(float)); + float *count = calloc(frames * PYSEG_TRACKS, sizeof(float)); + if (!count) { + failed = 1; + } else { + for (int i = 0; i < nthreads; ++i) { + const slide_worker *w = &workers[i]; + const size_t base = w->first_frame * PYSEG_TRACKS; + for (size_t j = 0; j < w->frame_count * PYSEG_TRACKS; ++j) { + tracks[base + j] += w->sum[j]; + count[base + j] += w->count[j]; + } + } + for (size_t j = 0; j < frames * PYSEG_TRACKS; ++j) + tracks[j] = count[j] > AGG_EPSILON ? tracks[j] / count[j] : 0.0f; + free(count); + } + } + + for (int i = 0; i < nthreads; ++i) { + free(workers[i].sum); + free(workers[i].count); + } + free(workers); + return failed ? 0 : frames; +} + +size_t pyseg_slide(const float *samples, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks) { + return slide_run(samples, NULL, num_samples, params, tracks); +} + +size_t pyseg_slide_s16(const int16_t *samples, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks) { + return slide_run(NULL, samples, num_samples, params, tracks); +} + +/* ---- binarization ----------------------------------------------------- + * + * Hysteresis over the aggregated score: a region opens when the score rises + * above onset and closes when it falls below offset, so a score hovering + * at the threshold does not chatter. The first frame decides the initial state + * on its own; every later frame is a possible transition, and a transition is + * timestamped at the frame that caused it. */ +size_t pyseg_binarize(const float *scores, + size_t num_frames, + int track, + const pyseg_binarize_params *params, + pyseg_segment *out, + size_t out_cap) { + pyseg_binarize_params defaults; + if (!params) { + pyseg_binarize_params_default(&defaults); + params = &defaults; + } + if (!scores || num_frames == 0 || track < 0 || track >= PYSEG_TRACKS) + return 0; + + const size_t cap = out ? out_cap : 0; + size_t n = 0; + int is_active = scores[(size_t)track] > params->onset; + double start = frame_middle(0); + double time = start; + + for (size_t f = 1; f < num_frames; ++f) { + const double value = scores[f * PYSEG_TRACKS + (size_t)track]; + time = frame_middle(f); + if (is_active) { + if (value < params->offset) { + if (n < cap) { + out[n].start = start; + out[n].end = time; + } + ++n; + is_active = 0; + } + } else if (value > params->onset) { + start = time; + is_active = 1; + } + } + if (is_active) { + if (n < cap) { + out[n].start = start; + out[n].end = time; + } + ++n; + } + + if (!out || n > cap) + return n; + + /* Both passes look across neighbours, so they run over the collected list + * rather than during the scan: a gap is only short once both of the + * segments around it are known. */ + if (params->min_duration_off > 0.0) { + size_t w = 0; + for (size_t i = 0; i < n; ++i) { + if (w > 0 && out[i].start - out[w - 1].end < params->min_duration_off) + out[w - 1].end = out[i].end; + else + out[w++] = out[i]; + } + n = w; + } + if (params->min_duration_on > 0.0) { + size_t w = 0; + for (size_t i = 0; i < n; ++i) + if (out[i].end - out[i].start >= params->min_duration_on) + out[w++] = out[i]; + n = w; + } + return n; +} diff --git a/src/wav.c b/src/wav.c new file mode 100644 index 0000000..2dcf27a --- /dev/null +++ b/src/wav.c @@ -0,0 +1,127 @@ +#include "wav.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static int rd_u32(FILE *f, uint32_t *v) { + unsigned char b[4]; + if (fread(b, 1, 4, f) != 4) + return -1; + *v = (uint32_t)b[0] | (uint32_t)b[1] << 8 | (uint32_t)b[2] << 16 | (uint32_t)b[3] << 24; + return 0; +} + +static int rd_u16(FILE *f, uint16_t *v) { + unsigned char b[2]; + if (fread(b, 1, 2, f) != 2) + return -1; + *v = (uint16_t)((uint32_t)b[0] | (uint32_t)b[1] << 8); + return 0; +} + +#define FAIL(msg) \ + do { \ + if (err) \ + *err = (msg); \ + goto fail; \ + } while (0) + +int wav_read_mono_s16(const char *path, wav_t *out, const char **err) { + memset(out, 0, sizeof *out); + FILE *f = fopen(path, "rb"); + if (!f) { + if (err) + *err = "cannot open file"; + return -1; + } + + char id[4]; + uint32_t riff_size = 0; + if (fread(id, 1, 4, f) != 4 || memcmp(id, "RIFF", 4) != 0) + FAIL("not a RIFF file"); + if (rd_u32(f, &riff_size) != 0) + FAIL("truncated header"); + if (fread(id, 1, 4, f) != 4 || memcmp(id, "WAVE", 4) != 0) + FAIL("not a WAVE file"); + + uint16_t fmt = 0, channels = 0, bits = 0; + uint32_t rate = 0; + int have_fmt = 0; + + for (;;) { + uint32_t size; + if (fread(id, 1, 4, f) != 4 || rd_u32(f, &size) != 0) + break; /* end of file, or a truncated trailing chunk */ + + if (memcmp(id, "fmt ", 4) == 0) { + uint16_t block_align; + uint32_t byte_rate; + if (size < 16) + FAIL("short fmt chunk"); + if (rd_u16(f, &fmt) || rd_u16(f, &channels) || rd_u32(f, &rate) || + rd_u32(f, &byte_rate) || rd_u16(f, &block_align) || rd_u16(f, &bits)) + FAIL("truncated fmt chunk"); + if (fseek(f, (long)size - 16, SEEK_CUR) != 0) + FAIL("bad fmt chunk size"); + have_fmt = 1; + continue; + } + + if (memcmp(id, "data", 4) == 0) { + if (!have_fmt) + FAIL("data chunk before fmt chunk"); + /* Format 1 is PCM; 0xFFFE is extensible, whose subformat this + * reader does not inspect, so it is accepted only at 16 bits. */ + if ((fmt != 1 && fmt != 0xFFFE) || bits != 16) + FAIL("only 16-bit PCM is supported"); + if (channels == 0) + FAIL("zero channels"); + + const size_t frames = size / (2u * channels); + if (frames == 0) + FAIL("empty data chunk"); + int16_t *mono = malloc(frames * sizeof(int16_t)); + if (!mono) + FAIL("out of memory"); + + const size_t step = channels; + int16_t buf[1024]; + size_t got = 0; + while (got < frames) { + const size_t want = (frames - got) < (sizeof buf / sizeof buf[0]) / step + ? (frames - got) * step + : (sizeof buf / sizeof buf[0]) / step * step; + const size_t n = fread(buf, sizeof(int16_t), want, f); + if (n < step) + break; + for (size_t i = 0; i + step <= n; i += step) + mono[got++] = buf[i]; + } + if (got == 0) { + free(mono); + FAIL("no samples read"); + } + out->samples = mono; + out->count = got; + out->sample_rate = (int)rate; + out->channels = (int)channels; + fclose(f); + return 0; + } + + /* Skip unknown chunks; RIFF chunks are word aligned. */ + if (fseek(f, (long)size + (size & 1u), SEEK_CUR) != 0) + break; + } + FAIL("no data chunk"); + +fail: + fclose(f); + return -1; +} + +void wav_free(wav_t *w) { + free(w->samples); + memset(w, 0, sizeof *w); +} diff --git a/src/wav.h b/src/wav.h new file mode 100644 index 0000000..859b535 --- /dev/null +++ b/src/wav.h @@ -0,0 +1,22 @@ +/* Minimal RIFF/WAVE reader for the command line tool: uncompressed 16-bit PCM + * only, first channel extracted. The library itself never touches files. */ + +#ifndef PYSEG_WAV_H +#define PYSEG_WAV_H + +#include <stddef.h> +#include <stdint.h> + +typedef struct { + int16_t *samples; /* mono, first channel of the source */ + size_t count; + int sample_rate; + int channels; +} wav_t; + +/* Returns 0 on success. On failure returns non-zero and, when err is + * non-NULL, writes a short reason there. */ +int wav_read_mono_s16(const char *path, wav_t *out, const char **err); +void wav_free(wav_t *w); + +#endif /* PYSEG_WAV_H */ diff --git a/src/weights.c b/src/weights.c new file mode 100644 index 0000000..42e3c9c --- /dev/null +++ b/src/weights.c @@ -0,0 +1,189 @@ +/* Weight blob reader. + * + * The file is a header, a fixed-size table of named tensors, then the tensor + * data at 64-byte boundaries. Parsing checks that every tensor the engine + * needs is present with the exact byte count that shape implies, so a blob + * exported from a different checkpoint or a different layout is rejected at + * load time rather than producing plausible wrong numbers. */ + +#include "weights.h" + +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <stdint.h> + +#define MAGIC "PSG1" +#define VERSION 1u +#define NAME_LEN 24 + +typedef struct { + char magic[4]; + uint32_t version; + uint32_t count; + uint32_t total; +} blob_header; + +typedef struct { + char name[NAME_LEN]; + uint32_t offset; + uint32_t nbytes; + uint32_t digest; +} blob_entry; + +typedef struct { + const unsigned char *base; + size_t len; + const blob_entry *table; + uint32_t count; + int verify; + int failed; +} reader; + +static uint32_t fnv1a(const unsigned char *p, size_t n) { + uint32_t h = 0x811C9DC5u; + for (size_t i = 0; i < n; ++i) + h = (h ^ p[i]) * 0x01000193u; + return h; +} + +/* Looks up one tensor and checks it is exactly floats long. A miss sets the + * reader's failure flag rather than returning it, so the caller can list every + * tensor once and test the flag at the end. */ +static const float *take(reader *r, const char *name, size_t floats) { + if (r->failed) + return NULL; + for (uint32_t i = 0; i < r->count; ++i) { + const blob_entry *e = &r->table[i]; + if (strncmp(e->name, name, NAME_LEN) != 0) + continue; + if (e->nbytes != floats * sizeof(float) || e->offset % 64 != 0 || e->offset > r->len || + e->nbytes > r->len - e->offset) + break; + const unsigned char *data = r->base + e->offset; + if (r->verify && fnv1a(data, e->nbytes) != e->digest) + break; + return (const float *)(const void *)data; + } + r->failed = 1; + return NULL; +} + +int pyseg_weights_parse(const void *blob, size_t len, int verify, pyseg_weights *out) { + if (!blob || len < sizeof(blob_header)) + return -1; + const blob_header *h = (const blob_header *)blob; + if (memcmp(h->magic, MAGIC, 4) != 0 || h->version != VERSION) + return -1; + if (h->count > 1024) + return -1; + const size_t table_bytes = (size_t)h->count * sizeof(blob_entry); + if (len < sizeof(blob_header) + table_bytes || len < h->total) + return -1; + + reader r = { + .base = (const unsigned char *)blob, + .len = len, + .table = (const blob_entry *)(const void *)((const unsigned char *)blob + sizeof *h), + .count = h->count, + .verify = verify, + .failed = 0, + }; + + memset(out, 0, sizeof *out); + out->wavnorm = take(&r, "wavnorm", 2); + out->sinc_w = take(&r, "sinc_w", 80 * 256); + + static const int norm_ch[3] = {80, 60, 60}; + for (int i = 0; i < 3; ++i) { + char name[NAME_LEN]; + snprintf(name, sizeof name, "norm%d_w", i + 1); + out->norm_w[i] = take(&r, name, (size_t)norm_ch[i]); + snprintf(name, sizeof name, "norm%d_b", i + 1); + out->norm_b[i] = take(&r, name, (size_t)norm_ch[i]); + } + + static const int conv_k[2] = {400, 300}; + for (int i = 0; i < 2; ++i) { + char name[NAME_LEN]; + snprintf(name, sizeof name, "conv%d_w", i + 2); + out->conv_w[i] = take(&r, name, 60u * (size_t)conv_k[i]); + snprintf(name, sizeof name, "conv%d_b", i + 2); + out->conv_b[i] = take(&r, name, 60); + } + + for (int l = 0; l < 4; ++l) { + const size_t in_dim = l == 0 ? 60 : 256; + for (int d = 0; d < 2; ++d) { + char name[NAME_LEN]; + const char tag = d == 0 ? 'f' : 'b'; + snprintf(name, sizeof name, "lstm%d%c_ih", l, tag); + out->lstm[l][d].ih = take(&r, name, 512 * in_dim); + snprintf(name, sizeof name, "lstm%d%c_hh", l, tag); + out->lstm[l][d].hh = take(&r, name, 512 * 128); + snprintf(name, sizeof name, "lstm%d%c_bias", l, tag); + out->lstm[l][d].bias = take(&r, name, 512); + } + } + + static const int lin_k[2] = {256, 128}; + for (int i = 0; i < 2; ++i) { + char name[NAME_LEN]; + snprintf(name, sizeof name, "lin%d_w", i); + out->lin_w[i] = take(&r, name, 128u * (size_t)lin_k[i]); + snprintf(name, sizeof name, "lin%d_b", i); + out->lin_b[i] = take(&r, name, 128); + } + + out->cls_w = take(&r, "cls_w", 7 * 128); + out->cls_b = take(&r, "cls_b", 7); + + return r.failed ? -1 : 0; +} + +int pyseg_weights_map(const char *path, int verify, pyseg_weights *out, void **map, size_t *map_len) { + const int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size <= 0) { + close(fd); + return -1; + } + const size_t len = (size_t)st.st_size; + void *addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (addr == MAP_FAILED) + return -1; + if (pyseg_weights_parse(addr, len, verify, out) != 0) { + munmap(addr, len); + return -1; + } + *map = addr; + *map_len = len; + return 0; +} + +void pyseg_weights_unmap(void *map, size_t len) { + if (map) + munmap(map, len); +} + +#ifdef PYSEG_EMBED_WEIGHTS +extern const unsigned char pyseg_weights_blob[]; +extern const size_t pyseg_weights_blob_len; + +const void *pyseg_weights_embedded(size_t *len) { + *len = pyseg_weights_blob_len; + return pyseg_weights_blob; +} +#else +const void *pyseg_weights_embedded(size_t *len) { + *len = 0; + return NULL; +} +#endif diff --git a/src/weights.h b/src/weights.h new file mode 100644 index 0000000..f32cdfb --- /dev/null +++ b/src/weights.h @@ -0,0 +1,49 @@ +/* Access to the packed weight blob produced by tools/export_weights.py. + * + * The blob is already in the layout the kernels read, so loading it is a table + * walk that hands out pointers: no transposing, no unpacking, no allocation + * for the weights themselves. That holds both for the copy compiled into the + * binary and for a file, which is mapped read-only and shared between every + * process on the host that maps it. */ + +#ifndef PYSEG_WEIGHTS_H +#define PYSEG_WEIGHTS_H + +#include <stddef.h> + +#define PYSEG_ALIGN64 __attribute__((aligned(64))) + +/* Every weight matrix is [rows, window] row-major, matching pyseg_kernels. */ +typedef struct { + const float *ih; /* [512, input] */ + const float *hh; /* [512, 128] */ + const float *bias; /* [512], the two PyTorch bias vectors already summed */ +} pyseg_lstm_weights; + +typedef struct { + const float *wavnorm; /* [2]: InstanceNorm1d scale then shift */ + const float *sinc_w; /* [80, 256]: 251 taps then five zero columns */ + const float *norm_w[3]; /* [80], [60], [60] */ + const float *norm_b[3]; + const float *conv_w[2]; /* [60, 400] and [60, 300] */ + const float *conv_b[2]; + pyseg_lstm_weights lstm[4][2]; /* [layer][0 = forward, 1 = backward] */ + const float *lin_w[2]; /* [128, 256] and [128, 128] */ + const float *lin_b[2]; + const float *cls_w; /* [7, 128] */ + const float *cls_b; +} pyseg_weights; + +/* Returns 0 on success. verify recomputes the per-tensor checksum. */ +int pyseg_weights_parse(const void *blob, size_t len, int verify, pyseg_weights *out); + +/* Maps path read-only and parses it. On success the caller must keep the + * mapping alive for as long as the weights are used. */ +int pyseg_weights_map(const char *path, int verify, pyseg_weights *out, void **map, size_t *map_len); + +void pyseg_weights_unmap(void *map, size_t len); + +/* The blob compiled into this binary, or NULL when built without one. */ +const void *pyseg_weights_embedded(size_t *len); + +#endif /* PYSEG_WEIGHTS_H */ |
