diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:03:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:13:28 +0200 |
| commit | 64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch) | |
| tree | 85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /src/pyseg.c | |
| download | pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip | |
Init
Diffstat (limited to 'src/pyseg.c')
| -rw-r--r-- | src/pyseg.c | 503 |
1 files changed, 503 insertions, 0 deletions
diff --git a/src/pyseg.c b/src/pyseg.c new file mode 100644 index 0000000..6860d54 --- /dev/null +++ b/src/pyseg.c @@ -0,0 +1,503 @@ +/* pyannote/segmentation-3.0 forward pass. + * + * Per window: normalise the waveform, one 251-tap filterbank and two ordinary + * convolutions each followed by max-pooling and per-channel normalisation, a + * four-layer bidirectional LSTM over the resulting 589 frames, two feed-forward + * layers and a seven-way softmax. Every buffer is allocated once in + * pyseg_create and reused, so a window costs no allocation, no syscall and no + * lock. The working set is 5.4 MB of activations plus 6 MB of read-only + * weights, which are shared by every engine mapping the same blob. + * + * Activations are stored time-major throughout, so the taps a convolution + * needs for one output step are one contiguous span and the whole network, + * convolutions included, runs through the single kernel in kernels.h. + * + * The convolutions are computed in tiles whose outputs are max-pooled on the + * spot. Only the pooled result is ever stored: the filterbank produces 15975 + * steps of 80 channels, 5.1 MB, of which only the 1.7 MB pooled form is needed + * downstream, and the same holds for the two later stages. */ + +#include "pyseg.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#include "kernels.h" +#include "weights.h" + +/* Stage geometry. Each convolution is followed by max-pooling by three, so + * only a multiple of three of its output steps is ever used; the last one or + * two steps of the later stages fall off the end and are not computed. */ +#define SINC_K 256 /* 251 taps padded with five zero weights */ +#define SINC_HOP 10 /* filterbank stride */ +#define C1 80 /* filterbank channels */ +#define T1 15975 /* filterbank output steps */ +#define P1 5325 /* after pooling */ + +#define C2 60 +#define K2 400 /* five taps of 80 channels */ +#define T2 5319 /* of 5321 produced by a valid kernel-5 convolution */ +#define P2 1773 + +#define C3 60 +#define K3 300 /* five taps of 60 channels */ +#define T3 1767 /* of 1769 */ +#define P3 PYSEG_FRAMES + +#define HID 128 +#define GATES (4 * HID) +#define BIDIR (2 * HID) + +/* Output steps per convolution tile. A multiple of three so pooling groups + * never straddle a tile boundary, and small enough that the tile stays in L1 + * while the weights stream past. */ +#define TILE 384 + +#define INSTANCE_NORM_EPS 1e-5 +#define LEAKY_SLOPE 0.01f + +/* Per-window buffer lengths, in floats. Everything before the LSTM is scratch + * shared by the windows of a batch, because they run through the front end one + * at a time; from the LSTM on, a batch is processed together and each window + * needs its own. */ +#define FEAT_STRIDE ((size_t)PYSEG_FRAMES * C3) +#define PRE_STRIDE ((size_t)PYSEG_FRAMES * GATES) +#define ACT_STRIDE ((size_t)PYSEG_FRAMES * BIDIR) + +struct pyseg { + const pyseg_kernels *k; + pyseg_weights w; + int ftz; + int batch; + + void *map; + size_t map_len; + size_t arena_bytes; + + void *arena; + float *wav; /* [PYSEG_WINDOW_SAMPLES + 8] */ + float *pool1; /* [P1][C1] */ + float *pool2; /* [P2][C2] */ + float *tile; /* [TILE][C1] */ + float *feat; /* [batch][P3][C3] */ + float *pre; /* [batch][PYSEG_FRAMES][GATES] */ + float *acta; /* [batch][PYSEG_FRAMES][BIDIR] */ + float *actb; /* [batch][PYSEG_FRAMES][BIDIR] */ + float *gates; /* [batch][GATES] */ + float *state; /* [2][batch][HID]: hidden then cell */ +}; + +/* ---- denormal handling ------------------------------------------------ + * + * The LSTM cell state decays toward zero across long silences, and denormal + * operands are slow on several targets. Flushing them is scoped to the call so + * the library never leaves the caller's floating point mode altered. */ + +typedef struct { + unsigned long long saved; + int active; +} fp_scope; + +#if defined(__aarch64__) +# define FPCR_FZ (1ull << 24) +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->active = enable; + if (!enable) + return; + unsigned long long r; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(r)); + s->saved = r; + if (!(r & FPCR_FZ)) + __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ)); +} +static inline void fp_scope_leave(const fp_scope *s) { + if (s->active) + __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved)); +} +#elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64) +# include <xmmintrin.h> +/* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring + * denormals-are-zero bit is left alone: it only affects denormal inputs, and + * it is absent on the earliest x86-64 parts, where writing it would fault. */ +# define MXCSR_FTZ 0x8000u +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->active = enable; + if (!enable) + return; + s->saved = _mm_getcsr(); + _mm_setcsr((unsigned)s->saved | MXCSR_FTZ); +} +static inline void fp_scope_leave(const fp_scope *s) { + if (s->active) + _mm_setcsr((unsigned)s->saved); +} +#else +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->saved = 0; + s->active = enable; +} +static inline void fp_scope_leave(const fp_scope *s) { (void)s; } +#endif + +/* ---- pointwise ------------------------------------------------------- */ + +static inline float leaky_(float x) { return x >= 0.0f ? x : LEAKY_SLOPE * x; } + +/* ---- normalisation ---------------------------------------------------- + * + * InstanceNorm1d over the whole window, one statistic pair per channel. The + * moments are accumulated in double and the deviations in a second pass: the + * filterbank output runs to five significant digits over 5325 steps, and a + * single-pass float variance loses the low ones to cancellation. + * + * This is also why the model cannot be run as a stream. Every frame's value + * depends on statistics of the entire ten-second window. */ +#define MAX_CHANNELS C1 + +static void instance_norm_leaky(float *x, int steps, int ch, const float *scale, const float *shift) { + double mean[MAX_CHANNELS], inv[MAX_CHANNELS]; + + for (int c = 0; c < ch; ++c) + mean[c] = 0.0; + for (int t = 0; t < steps; ++t) { + const float *row = x + (size_t)t * ch; + for (int c = 0; c < ch; ++c) + mean[c] += row[c]; + } + for (int c = 0; c < ch; ++c) + mean[c] /= steps; + + for (int c = 0; c < ch; ++c) + inv[c] = 0.0; + for (int t = 0; t < steps; ++t) { + const float *row = x + (size_t)t * ch; + for (int c = 0; c < ch; ++c) { + const double d = row[c] - mean[c]; + inv[c] += d * d; + } + } + for (int c = 0; c < ch; ++c) + inv[c] = 1.0 / sqrt(inv[c] / steps + INSTANCE_NORM_EPS); + + for (int t = 0; t < steps; ++t) { + float *row = x + (size_t)t * ch; + for (int c = 0; c < ch; ++c) + row[c] = leaky_((float)((row[c] - mean[c]) * inv[c] * scale[c] + shift[c])); + } +} + +/* ---- convolution stages ---------------------------------------------- + * + * One call covers a whole stage: convolution, optional magnitude, pooling by + * three, then the channel statistics. lda is how far apart consecutive + * output windows start in the source, which is the filterbank's hop for the + * first stage and one frame for the later ones. */ +static void conv_pool_norm(pyseg_t *v, + float *dst, + const float *src, + int lda, + const float *weight, + const float *bias, + int steps, + int ch, + int k, + int take_abs, + const float *scale, + const float *shift) { + for (int t0 = 0; t0 < steps; t0 += TILE) { + const int n = steps - t0 < TILE ? steps - t0 : TILE; + v->k->gemm_nt(v->tile, ch, src + (size_t)t0 * lda, lda, weight, bias, n, ch, k); + + float *out = dst + (size_t)(t0 / 3) * ch; + for (int p = 0; p < n / 3; ++p) { + const float *a = v->tile + (size_t)(3 * p) * ch; + const float *b = a + ch, *c = b + ch; + float *o = out + (size_t)p * ch; + if (take_abs) { + for (int i = 0; i < ch; ++i) { + const float x = fabsf(a[i]), y = fabsf(b[i]), z = fabsf(c[i]); + o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z); + } + } else { + for (int i = 0; i < ch; ++i) { + const float x = a[i], y = b[i], z = c[i]; + o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z); + } + } + } + } + instance_norm_leaky(dst, steps / 3, ch, scale, shift); +} + +/* ---- LSTM ------------------------------------------------------------- + * + * The input projection of every time step shares one weight matrix, so it runs + * as a single pass over every step of every window in the batch before the + * recurrence starts, which is three fifths of the layer's arithmetic done at + * full matrix-multiply efficiency. + * + * What is left is the hidden-to-hidden product, one per step, and it is the + * reason batching exists. On its own it reads a 256 KB weight matrix to + * perform 65536 multiply-adds, so it runs at the speed of memory rather than + * of arithmetic and is the single most expensive part of a window. Windows are + * independent, so running several of them in step turns that matrix-vector + * product into a small matrix-matrix product: one weight load then serves + * every window, and the recurrence speeds up by better than a factor of two at + * a batch of four. */ +static void lstm_direction(pyseg_t *v, + int count, + const float *in, + int lda, + int k, + const pyseg_lstm_weights *w, + float *out, + int offset, + int reverse) { + v->k->gemm_nt(v->pre, GATES, in, lda, w->ih, w->bias, count * PYSEG_FRAMES, GATES, k); + + float *h = v->state, *c = v->state + (size_t)v->batch * HID; + memset(h, 0, (size_t)count * HID * sizeof(float)); + memset(c, 0, (size_t)count * HID * sizeof(float)); + + for (int step = 0; step < PYSEG_FRAMES; ++step) { + const int t = reverse ? PYSEG_FRAMES - 1 - step : step; + v->k->gemm_nt(v->gates, GATES, h, HID, w->hh, NULL, count, GATES, HID); + for (int b = 0; b < count; ++b) { + float *hb = h + (size_t)b * HID; + v->k->lstm_cell(hb, c + (size_t)b * HID, v->gates + (size_t)b * GATES, + v->pre + (size_t)b * PRE_STRIDE + (size_t)t * GATES, HID); + memcpy(out + (size_t)b * ACT_STRIDE + (size_t)t * BIDIR + offset, hb, + HID * sizeof(float)); + } + } +} + +/* ---- forward pass ---------------------------------------------------- */ + +/* Everything up to the LSTM, for one window of the batch. */ +static void front_end(pyseg_t *v, int slot) { + const pyseg_weights *w = &v->w; + + /* Waveform normalisation: one channel, statistics over the whole window. */ + { + double mean = 0.0, var = 0.0; + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) + mean += v->wav[i]; + mean /= PYSEG_WINDOW_SAMPLES; + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) { + const double d = v->wav[i] - mean; + var += d * d; + } + const double inv = 1.0 / sqrt(var / PYSEG_WINDOW_SAMPLES + INSTANCE_NORM_EPS); + const float scale = w->wavnorm[0], shift = w->wavnorm[1]; + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) + v->wav[i] = (float)((v->wav[i] - mean) * inv) * scale + shift; + } + + conv_pool_norm(v, v->pool1, v->wav, SINC_HOP, w->sinc_w, NULL, T1, C1, SINC_K, 1, w->norm_w[0], + w->norm_b[0]); + conv_pool_norm(v, v->pool2, v->pool1, C1, w->conv_w[0], w->conv_b[0], T2, C2, K2, 0, w->norm_w[1], + w->norm_b[1]); + conv_pool_norm(v, v->feat + (size_t)slot * FEAT_STRIDE, v->pool2, C2, w->conv_w[1], w->conv_b[1], + T3, C3, K3, 0, w->norm_w[2], w->norm_b[2]); +} + +/* The LSTM stack, the feed-forward layers and the softmax, over a whole batch + * whose features the front end has already produced. */ +static void back_end(pyseg_t *v, int count, float *probs) { + const pyseg_weights *w = &v->w; + const pyseg_kernels *k = v->k; + const int rows = count * PYSEG_FRAMES; + + lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][0], v->acta, 0, 0); + lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][1], v->acta, HID, 1); + for (int layer = 1; layer < 4; ++layer) { + const float *in = (layer & 1) ? v->acta : v->actb; + float *out = (layer & 1) ? v->actb : v->acta; + lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][0], out, 0, 0); + lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][1], out, HID, 1); + } + + /* Layer 3 wrote actb; the feed-forward stack then ping-pongs back. Both + * layers are narrower than what they read, so writing over the buffer the + * previous layer occupied is safe. */ + k->gemm_nt(v->acta, HID, v->actb, BIDIR, w->lin_w[0], w->lin_b[0], rows, HID, BIDIR); + for (int i = 0; i < rows * HID; ++i) + v->acta[i] = leaky_(v->acta[i]); + k->gemm_nt(v->actb, HID, v->acta, HID, w->lin_w[1], w->lin_b[1], rows, HID, HID); + for (int i = 0; i < rows * HID; ++i) + v->actb[i] = leaky_(v->actb[i]); + + k->gemm_nt(probs, PYSEG_CLASSES, v->actb, HID, w->cls_w, w->cls_b, rows, PYSEG_CLASSES, HID); + + for (int t = 0; t < rows; ++t) { + float *row = probs + (size_t)t * PYSEG_CLASSES; + float max = row[0]; + for (int i = 1; i < PYSEG_CLASSES; ++i) + if (row[i] > max) + max = row[i]; + float sum = 0.0f; + for (int i = 0; i < PYSEG_CLASSES; ++i) { + row[i] = expf(row[i] - max); + sum += row[i]; + } + const float inv = 1.0f / sum; + for (int i = 0; i < PYSEG_CLASSES; ++i) + row[i] *= inv; + } +} + +/* ---- lifecycle ------------------------------------------------------- */ + +pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch) { + if (batch < 1 || batch > PYSEG_MAX_BATCH) + return NULL; + + pyseg_t *v = calloc(1, sizeof *v); + if (!v) + return NULL; + v->batch = batch; + + if (path) { + const int verify = !(flags & PYSEG_NO_VERIFY); + if (pyseg_weights_map(path, verify, &v->w, &v->map, &v->map_len) != 0) { + free(v); + return NULL; + } + } else { + size_t len = 0; + const void *blob = pyseg_weights_embedded(&len); + if (!blob || pyseg_weights_parse(blob, len, 0, &v->w) != 0) { + free(v); + return NULL; + } + } + + /* One allocation, carved into 64-byte aligned buffers in this order. */ + const size_t b = (size_t)batch; + const size_t sizes[] = { + PYSEG_WINDOW_SAMPLES + 8, /* wav, with slack past the last filter window */ + (size_t)P1 * C1, /* pool1 */ + (size_t)P2 * C2, /* pool2 */ + (size_t)TILE * C1, /* tile */ + b * FEAT_STRIDE, /* feat */ + b * PRE_STRIDE, /* pre */ + b * ACT_STRIDE, /* acta */ + b * ACT_STRIDE, /* actb */ + b * GATES, /* gates */ + 2 * b * HID, /* state */ + }; + float **slots[] = {&v->wav, &v->pool1, &v->pool2, &v->tile, &v->feat, + &v->pre, &v->acta, &v->actb, &v->gates, &v->state}; + + size_t total = 0; + for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) + total += (sizes[i] * sizeof(float) + 63) / 64 * 64; + if (posix_memalign(&v->arena, 64, total) != 0 || !v->arena) { + pyseg_weights_unmap(v->map, v->map_len); + free(v); + return NULL; + } + memset(v->arena, 0, total); + v->arena_bytes = total; + + unsigned char *p = v->arena; + for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) { + *slots[i] = (float *)(void *)p; + p += (sizes[i] * sizeof(float) + 63) / 64 * 64; + } + + v->k = (flags & PYSEG_SCALAR_KERNELS) ? pyseg_kernels_scalar() : pyseg_kernels_simd(); + v->ftz = (flags & PYSEG_NO_FTZ) == 0; + return v; +} + +pyseg_t *pyseg_create_ex(const char *path, unsigned flags) { + return pyseg_create_batch(path, flags, 1); +} + +pyseg_t *pyseg_create(void) { return pyseg_create_ex(NULL, 0); } + +pyseg_t *pyseg_create_from_file(const char *path) { return pyseg_create_ex(path, 0); } + +void pyseg_destroy(pyseg_t *v) { + if (!v) + return; + pyseg_weights_unmap(v->map, v->map_len); + free(v->arena); + free(v); +} + +const char *pyseg_backend_name(const pyseg_t *v) { return v->k->name; } + +int pyseg_batch(const pyseg_t *v) { return v->batch; } + +size_t pyseg_memory_bytes(const pyseg_t *v) { + return sizeof *v + v->arena_bytes + (v->map ? v->map_len : 0); +} + +/* ---- entry points ----------------------------------------------------- */ + +void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs) { + if (count < 1 || count > v->batch) + return; + fp_scope fp; + fp_scope_enter(&fp, v->ftz); + for (int b = 0; b < count; ++b) { + memcpy(v->wav, samples + (size_t)b * PYSEG_WINDOW_SAMPLES, + PYSEG_WINDOW_SAMPLES * sizeof(float)); + front_end(v, b); + } + back_end(v, count, probs); + fp_scope_leave(&fp); +} + +void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs) { + pyseg_process_batch_f32(v, samples, 1, probs); +} + +void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs) { + fp_scope fp; + fp_scope_enter(&fp, v->ftz); + for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) + v->wav[i] = (float)samples[i] * (1.0f / 32768.0f); + front_end(v, 0); + back_end(v, 1, probs); + fp_scope_leave(&fp); +} + +/* ---- powerset decoding ----------------------------------------------- + * + * The seven classes are, in order, the empty set, the three singletons and the + * three pairs, which is the order itertools.combinations produces and the + * order the classifier was trained in. */ +static const unsigned char powerset_members[PYSEG_CLASSES] = { + 0x0, 0x1, 0x2, 0x4, 0x3, 0x5, 0x6, +}; + +void pyseg_to_multilabel(const float *probs, float *active) { + for (int t = 0; t < PYSEG_FRAMES; ++t) { + const float *row = probs + (size_t)t * PYSEG_CLASSES; + int best = 0; + for (int i = 1; i < PYSEG_CLASSES; ++i) + if (row[i] > row[best]) + best = i; + const unsigned mask = powerset_members[best]; + float *out = active + (size_t)t * PYSEG_SPEAKERS; + for (int s = 0; s < PYSEG_SPEAKERS; ++s) + out[s] = (mask >> s) & 1u ? 1.0f : 0.0f; + } +} + +int pyseg_powerset_class(int speaker_a, int speaker_b) { + unsigned mask = 0; + if (speaker_a >= 0) + mask |= 1u << speaker_a; + if (speaker_b >= 0) + mask |= 1u << speaker_b; + for (int i = 0; i < PYSEG_CLASSES; ++i) + if (powerset_members[i] == mask) + return i; + return -1; +} |
