/* pyannote/segmentation-3.0 forward pass. * * Per window: normalise the waveform, one 251-tap filterbank and two ordinary * convolutions each followed by max-pooling and per-channel normalisation, a * four-layer bidirectional LSTM over the resulting 589 frames, two feed-forward * layers and a seven-way softmax. Every buffer is allocated once in * pyseg_create and reused, so a window costs no allocation, no syscall and no * lock. The working set is 5.4 MB of activations plus 6 MB of read-only * weights, which are shared by every engine mapping the same blob. * * Activations are stored time-major throughout, so the taps a convolution * needs for one output step are one contiguous span and the whole network, * convolutions included, runs through the single kernel in kernels.h. * * The convolutions are computed in tiles whose outputs are max-pooled on the * spot. Only the pooled result is ever stored: the filterbank produces 15975 * steps of 80 channels, 5.1 MB, of which only the 1.7 MB pooled form is needed * downstream, and the same holds for the two later stages. */ #include "pyseg.h" #include #include #include #include "kernels.h" #include "weights.h" /* Stage geometry. Each convolution is followed by max-pooling by three, so * only a multiple of three of its output steps is ever used; the last one or * two steps of the later stages fall off the end and are not computed. */ #define SINC_K 256 /* 251 taps padded with five zero weights */ #define SINC_HOP 10 /* filterbank stride */ #define C1 80 /* filterbank channels */ #define T1 15975 /* filterbank output steps */ #define P1 5325 /* after pooling */ #define C2 60 #define K2 400 /* five taps of 80 channels */ #define T2 5319 /* of 5321 produced by a valid kernel-5 convolution */ #define P2 1773 #define C3 60 #define K3 300 /* five taps of 60 channels */ #define T3 1767 /* of 1769 */ #define P3 PYSEG_FRAMES #define HID 128 #define GATES (4 * HID) #define BIDIR (2 * HID) /* Output steps per convolution tile. A multiple of three so pooling groups * never straddle a tile boundary, and small enough that the tile stays in L1 * while the weights stream past. */ #define TILE 384 #define INSTANCE_NORM_EPS 1e-5 #define LEAKY_SLOPE 0.01f /* Per-window buffer lengths, in floats. Everything before the LSTM is scratch * shared by the windows of a batch, because they run through the front end one * at a time; from the LSTM on, a batch is processed together and each window * needs its own. */ #define FEAT_STRIDE ((size_t)PYSEG_FRAMES * C3) #define PRE_STRIDE ((size_t)PYSEG_FRAMES * GATES) #define ACT_STRIDE ((size_t)PYSEG_FRAMES * BIDIR) struct pyseg { const pyseg_kernels *k; pyseg_weights w; int ftz; int batch; void *map; size_t map_len; size_t arena_bytes; void *arena; float *wav; /* [PYSEG_WINDOW_SAMPLES + 8] */ float *pool1; /* [P1][C1] */ float *pool2; /* [P2][C2] */ float *tile; /* [TILE][C1] */ float *feat; /* [batch][P3][C3] */ float *pre; /* [batch][PYSEG_FRAMES][GATES] */ float *acta; /* [batch][PYSEG_FRAMES][BIDIR] */ float *actb; /* [batch][PYSEG_FRAMES][BIDIR] */ float *gates; /* [batch][GATES] */ float *state; /* [2][batch][HID]: hidden then cell */ }; /* ---- denormal handling ------------------------------------------------ * * The LSTM cell state decays toward zero across long silences, and denormal * operands are slow on several targets. Flushing them is scoped to the call so * the library never leaves the caller's floating point mode altered. */ typedef struct { unsigned long long saved; int active; } fp_scope; #if defined(__aarch64__) # define FPCR_FZ (1ull << 24) static inline void fp_scope_enter(fp_scope *s, int enable) { s->active = enable; if (!enable) return; unsigned long long r; __asm__ __volatile__("mrs %0, fpcr" : "=r"(r)); s->saved = r; if (!(r & FPCR_FZ)) __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ)); } static inline void fp_scope_leave(const fp_scope *s) { if (s->active) __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved)); } #elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64) # include /* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring * denormals-are-zero bit is left alone: it only affects denormal inputs, and * it is absent on the earliest x86-64 parts, where writing it would fault. */ # define MXCSR_FTZ 0x8000u static inline void fp_scope_enter(fp_scope *s, int enable) { s->active = enable; if (!enable) return; s->saved = _mm_getcsr(); _mm_setcsr((unsigned)s->saved | MXCSR_FTZ); } static inline void fp_scope_leave(const fp_scope *s) { if (s->active) _mm_setcsr((unsigned)s->saved); } #else static inline void fp_scope_enter(fp_scope *s, int enable) { s->saved = 0; s->active = enable; } static inline void fp_scope_leave(const fp_scope *s) { (void)s; } #endif /* ---- pointwise ------------------------------------------------------- */ static inline float leaky_(float x) { return x >= 0.0f ? x : LEAKY_SLOPE * x; } /* ---- normalisation ---------------------------------------------------- * * InstanceNorm1d over the whole window, one statistic pair per channel. The * moments are accumulated in double and the deviations in a second pass: the * filterbank output runs to five significant digits over 5325 steps, and a * single-pass float variance loses the low ones to cancellation. * * This is also why the model cannot be run as a stream. Every frame's value * depends on statistics of the entire ten-second window. */ #define MAX_CHANNELS C1 static void instance_norm_leaky(float *x, int steps, int ch, const float *scale, const float *shift) { double mean[MAX_CHANNELS], inv[MAX_CHANNELS]; for (int c = 0; c < ch; ++c) mean[c] = 0.0; for (int t = 0; t < steps; ++t) { const float *row = x + (size_t)t * ch; for (int c = 0; c < ch; ++c) mean[c] += row[c]; } for (int c = 0; c < ch; ++c) mean[c] /= steps; for (int c = 0; c < ch; ++c) inv[c] = 0.0; for (int t = 0; t < steps; ++t) { const float *row = x + (size_t)t * ch; for (int c = 0; c < ch; ++c) { const double d = row[c] - mean[c]; inv[c] += d * d; } } for (int c = 0; c < ch; ++c) inv[c] = 1.0 / sqrt(inv[c] / steps + INSTANCE_NORM_EPS); for (int t = 0; t < steps; ++t) { float *row = x + (size_t)t * ch; for (int c = 0; c < ch; ++c) row[c] = leaky_((float)((row[c] - mean[c]) * inv[c] * scale[c] + shift[c])); } } /* ---- convolution stages ---------------------------------------------- * * One call covers a whole stage: convolution, optional magnitude, pooling by * three, then the channel statistics. lda is how far apart consecutive * output windows start in the source, which is the filterbank's hop for the * first stage and one frame for the later ones. */ static void conv_pool_norm(pyseg_t *v, float *dst, const float *src, int lda, const float *weight, const float *bias, int steps, int ch, int k, int take_abs, const float *scale, const float *shift) { for (int t0 = 0; t0 < steps; t0 += TILE) { const int n = steps - t0 < TILE ? steps - t0 : TILE; v->k->gemm_nt(v->tile, ch, src + (size_t)t0 * lda, lda, weight, bias, n, ch, k); float *out = dst + (size_t)(t0 / 3) * ch; for (int p = 0; p < n / 3; ++p) { const float *a = v->tile + (size_t)(3 * p) * ch; const float *b = a + ch, *c = b + ch; float *o = out + (size_t)p * ch; if (take_abs) { for (int i = 0; i < ch; ++i) { const float x = fabsf(a[i]), y = fabsf(b[i]), z = fabsf(c[i]); o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z); } } else { for (int i = 0; i < ch; ++i) { const float x = a[i], y = b[i], z = c[i]; o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z); } } } } instance_norm_leaky(dst, steps / 3, ch, scale, shift); } /* ---- LSTM ------------------------------------------------------------- * * The input projection of every time step shares one weight matrix, so it runs * as a single pass over every step of every window in the batch before the * recurrence starts, which is three fifths of the layer's arithmetic done at * full matrix-multiply efficiency. * * What is left is the hidden-to-hidden product, one per step, and it is the * reason batching exists. On its own it reads a 256 KB weight matrix to * perform 65536 multiply-adds, so it runs at the speed of memory rather than * of arithmetic and is the single most expensive part of a window. Windows are * independent, so running several of them in step turns that matrix-vector * product into a small matrix-matrix product: one weight load then serves * every window, and the recurrence speeds up by better than a factor of two at * a batch of four. */ static void lstm_direction(pyseg_t *v, int count, const float *in, int lda, int k, const pyseg_lstm_weights *w, float *out, int offset, int reverse) { v->k->gemm_nt(v->pre, GATES, in, lda, w->ih, w->bias, count * PYSEG_FRAMES, GATES, k); float *h = v->state, *c = v->state + (size_t)v->batch * HID; memset(h, 0, (size_t)count * HID * sizeof(float)); memset(c, 0, (size_t)count * HID * sizeof(float)); for (int step = 0; step < PYSEG_FRAMES; ++step) { const int t = reverse ? PYSEG_FRAMES - 1 - step : step; v->k->gemm_nt(v->gates, GATES, h, HID, w->hh, NULL, count, GATES, HID); for (int b = 0; b < count; ++b) { float *hb = h + (size_t)b * HID; v->k->lstm_cell(hb, c + (size_t)b * HID, v->gates + (size_t)b * GATES, v->pre + (size_t)b * PRE_STRIDE + (size_t)t * GATES, HID); memcpy(out + (size_t)b * ACT_STRIDE + (size_t)t * BIDIR + offset, hb, HID * sizeof(float)); } } } /* ---- forward pass ---------------------------------------------------- */ /* Everything up to the LSTM, for one window of the batch. */ static void front_end(pyseg_t *v, int slot) { const pyseg_weights *w = &v->w; /* Waveform normalisation: one channel, statistics over the whole window. */ { double mean = 0.0, var = 0.0; for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) mean += v->wav[i]; mean /= PYSEG_WINDOW_SAMPLES; for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) { const double d = v->wav[i] - mean; var += d * d; } const double inv = 1.0 / sqrt(var / PYSEG_WINDOW_SAMPLES + INSTANCE_NORM_EPS); const float scale = w->wavnorm[0], shift = w->wavnorm[1]; for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) v->wav[i] = (float)((v->wav[i] - mean) * inv) * scale + shift; } conv_pool_norm(v, v->pool1, v->wav, SINC_HOP, w->sinc_w, NULL, T1, C1, SINC_K, 1, w->norm_w[0], w->norm_b[0]); conv_pool_norm(v, v->pool2, v->pool1, C1, w->conv_w[0], w->conv_b[0], T2, C2, K2, 0, w->norm_w[1], w->norm_b[1]); conv_pool_norm(v, v->feat + (size_t)slot * FEAT_STRIDE, v->pool2, C2, w->conv_w[1], w->conv_b[1], T3, C3, K3, 0, w->norm_w[2], w->norm_b[2]); } /* The LSTM stack, the feed-forward layers and the softmax, over a whole batch * whose features the front end has already produced. */ static void back_end(pyseg_t *v, int count, float *probs) { const pyseg_weights *w = &v->w; const pyseg_kernels *k = v->k; const int rows = count * PYSEG_FRAMES; lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][0], v->acta, 0, 0); lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][1], v->acta, HID, 1); for (int layer = 1; layer < 4; ++layer) { const float *in = (layer & 1) ? v->acta : v->actb; float *out = (layer & 1) ? v->actb : v->acta; lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][0], out, 0, 0); lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][1], out, HID, 1); } /* Layer 3 wrote actb; the feed-forward stack then ping-pongs back. Both * layers are narrower than what they read, so writing over the buffer the * previous layer occupied is safe. */ k->gemm_nt(v->acta, HID, v->actb, BIDIR, w->lin_w[0], w->lin_b[0], rows, HID, BIDIR); for (int i = 0; i < rows * HID; ++i) v->acta[i] = leaky_(v->acta[i]); k->gemm_nt(v->actb, HID, v->acta, HID, w->lin_w[1], w->lin_b[1], rows, HID, HID); for (int i = 0; i < rows * HID; ++i) v->actb[i] = leaky_(v->actb[i]); k->gemm_nt(probs, PYSEG_CLASSES, v->actb, HID, w->cls_w, w->cls_b, rows, PYSEG_CLASSES, HID); for (int t = 0; t < rows; ++t) { float *row = probs + (size_t)t * PYSEG_CLASSES; float max = row[0]; for (int i = 1; i < PYSEG_CLASSES; ++i) if (row[i] > max) max = row[i]; float sum = 0.0f; for (int i = 0; i < PYSEG_CLASSES; ++i) { row[i] = expf(row[i] - max); sum += row[i]; } const float inv = 1.0f / sum; for (int i = 0; i < PYSEG_CLASSES; ++i) row[i] *= inv; } } /* ---- lifecycle ------------------------------------------------------- */ pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch) { if (batch < 1 || batch > PYSEG_MAX_BATCH) return NULL; pyseg_t *v = calloc(1, sizeof *v); if (!v) return NULL; v->batch = batch; if (path) { const int verify = !(flags & PYSEG_NO_VERIFY); if (pyseg_weights_map(path, verify, &v->w, &v->map, &v->map_len) != 0) { free(v); return NULL; } } else { size_t len = 0; const void *blob = pyseg_weights_embedded(&len); if (!blob || pyseg_weights_parse(blob, len, 0, &v->w) != 0) { free(v); return NULL; } } /* One allocation, carved into 64-byte aligned buffers in this order. */ const size_t b = (size_t)batch; const size_t sizes[] = { PYSEG_WINDOW_SAMPLES + 8, /* wav, with slack past the last filter window */ (size_t)P1 * C1, /* pool1 */ (size_t)P2 * C2, /* pool2 */ (size_t)TILE * C1, /* tile */ b * FEAT_STRIDE, /* feat */ b * PRE_STRIDE, /* pre */ b * ACT_STRIDE, /* acta */ b * ACT_STRIDE, /* actb */ b * GATES, /* gates */ 2 * b * HID, /* state */ }; float **slots[] = {&v->wav, &v->pool1, &v->pool2, &v->tile, &v->feat, &v->pre, &v->acta, &v->actb, &v->gates, &v->state}; size_t total = 0; for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) total += (sizes[i] * sizeof(float) + 63) / 64 * 64; if (posix_memalign(&v->arena, 64, total) != 0 || !v->arena) { pyseg_weights_unmap(v->map, v->map_len); free(v); return NULL; } memset(v->arena, 0, total); v->arena_bytes = total; unsigned char *p = v->arena; for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) { *slots[i] = (float *)(void *)p; p += (sizes[i] * sizeof(float) + 63) / 64 * 64; } v->k = (flags & PYSEG_SCALAR_KERNELS) ? pyseg_kernels_scalar() : pyseg_kernels_simd(); v->ftz = (flags & PYSEG_NO_FTZ) == 0; return v; } pyseg_t *pyseg_create_ex(const char *path, unsigned flags) { return pyseg_create_batch(path, flags, 1); } pyseg_t *pyseg_create(void) { return pyseg_create_ex(NULL, 0); } pyseg_t *pyseg_create_from_file(const char *path) { return pyseg_create_ex(path, 0); } void pyseg_destroy(pyseg_t *v) { if (!v) return; pyseg_weights_unmap(v->map, v->map_len); free(v->arena); free(v); } const char *pyseg_backend_name(const pyseg_t *v) { return v->k->name; } int pyseg_batch(const pyseg_t *v) { return v->batch; } size_t pyseg_memory_bytes(const pyseg_t *v) { return sizeof *v + v->arena_bytes + (v->map ? v->map_len : 0); } /* ---- entry points ----------------------------------------------------- */ void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs) { if (count < 1 || count > v->batch) return; fp_scope fp; fp_scope_enter(&fp, v->ftz); for (int b = 0; b < count; ++b) { memcpy(v->wav, samples + (size_t)b * PYSEG_WINDOW_SAMPLES, PYSEG_WINDOW_SAMPLES * sizeof(float)); front_end(v, b); } back_end(v, count, probs); fp_scope_leave(&fp); } void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs) { pyseg_process_batch_f32(v, samples, 1, probs); } void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs) { fp_scope fp; fp_scope_enter(&fp, v->ftz); for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) v->wav[i] = (float)samples[i] * (1.0f / 32768.0f); front_end(v, 0); back_end(v, 1, probs); fp_scope_leave(&fp); } /* ---- powerset decoding ----------------------------------------------- * * The seven classes are, in order, the empty set, the three singletons and the * three pairs, which is the order itertools.combinations produces and the * order the classifier was trained in. */ static const unsigned char powerset_members[PYSEG_CLASSES] = { 0x0, 0x1, 0x2, 0x4, 0x3, 0x5, 0x6, }; void pyseg_to_multilabel(const float *probs, float *active) { for (int t = 0; t < PYSEG_FRAMES; ++t) { const float *row = probs + (size_t)t * PYSEG_CLASSES; int best = 0; for (int i = 1; i < PYSEG_CLASSES; ++i) if (row[i] > row[best]) best = i; const unsigned mask = powerset_members[best]; float *out = active + (size_t)t * PYSEG_SPEAKERS; for (int s = 0; s < PYSEG_SPEAKERS; ++s) out[s] = (mask >> s) & 1u ? 1.0f : 0.0f; } } int pyseg_powerset_class(int speaker_a, int speaker_b) { unsigned mask = 0; if (speaker_a >= 0) mask |= 1u << speaker_a; if (speaker_b >= 0) mask |= 1u << speaker_b; for (int i = 0; i < PYSEG_CLASSES; ++i) if (powerset_members[i] == mask) return i; return -1; }