aboutsummaryrefslogtreecommitdiff
path: root/src/pyseg.c
blob: 6860d54933394add4b56730a1ecfdf850465d5d1 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
/* pyannote/segmentation-3.0 forward pass.
 *
 * Per window: normalise the waveform, one 251-tap filterbank and two ordinary
 * convolutions each followed by max-pooling and per-channel normalisation, a
 * four-layer bidirectional LSTM over the resulting 589 frames, two feed-forward
 * layers and a seven-way softmax. Every buffer is allocated once in
 * pyseg_create and reused, so a window costs no allocation, no syscall and no
 * lock. The working set is 5.4 MB of activations plus 6 MB of read-only
 * weights, which are shared by every engine mapping the same blob.
 *
 * Activations are stored time-major throughout, so the taps a convolution
 * needs for one output step are one contiguous span and the whole network,
 * convolutions included, runs through the single kernel in kernels.h.
 *
 * The convolutions are computed in tiles whose outputs are max-pooled on the
 * spot. Only the pooled result is ever stored: the filterbank produces 15975
 * steps of 80 channels, 5.1 MB, of which only the 1.7 MB pooled form is needed
 * downstream, and the same holds for the two later stages. */

#include "pyseg.h"

#include <math.h>
#include <stdlib.h>
#include <string.h>

#include "kernels.h"
#include "weights.h"

/* Stage geometry. Each convolution is followed by max-pooling by three, so
 * only a multiple of three of its output steps is ever used; the last one or
 * two steps of the later stages fall off the end and are not computed. */
#define SINC_K   256   /* 251 taps padded with five zero weights */
#define SINC_HOP 10    /* filterbank stride */
#define C1       80    /* filterbank channels */
#define T1       15975 /* filterbank output steps */
#define P1       5325  /* after pooling */

#define C2 60
#define K2 400  /* five taps of 80 channels */
#define T2 5319 /* of 5321 produced by a valid kernel-5 convolution */
#define P2 1773

#define C3 60
#define K3 300  /* five taps of 60 channels */
#define T3 1767 /* of 1769 */
#define P3 PYSEG_FRAMES

#define HID   128
#define GATES (4 * HID)
#define BIDIR (2 * HID)

/* Output steps per convolution tile. A multiple of three so pooling groups
 * never straddle a tile boundary, and small enough that the tile stays in L1
 * while the weights stream past. */
#define TILE 384

#define INSTANCE_NORM_EPS 1e-5
#define LEAKY_SLOPE       0.01f

/* Per-window buffer lengths, in floats. Everything before the LSTM is scratch
 * shared by the windows of a batch, because they run through the front end one
 * at a time; from the LSTM on, a batch is processed together and each window
 * needs its own. */
#define FEAT_STRIDE ((size_t)PYSEG_FRAMES * C3)
#define PRE_STRIDE  ((size_t)PYSEG_FRAMES * GATES)
#define ACT_STRIDE  ((size_t)PYSEG_FRAMES * BIDIR)

struct pyseg {
    const pyseg_kernels *k;
    pyseg_weights w;
    int ftz;
    int batch;

    void *map;
    size_t map_len;
    size_t arena_bytes;

    void *arena;
    float *wav;   /* [PYSEG_WINDOW_SAMPLES + 8] */
    float *pool1; /* [P1][C1] */
    float *pool2; /* [P2][C2] */
    float *tile;  /* [TILE][C1] */
    float *feat;  /* [batch][P3][C3] */
    float *pre;   /* [batch][PYSEG_FRAMES][GATES] */
    float *acta;  /* [batch][PYSEG_FRAMES][BIDIR] */
    float *actb;  /* [batch][PYSEG_FRAMES][BIDIR] */
    float *gates; /* [batch][GATES] */
    float *state; /* [2][batch][HID]: hidden then cell */
};

/* ---- denormal handling ------------------------------------------------
 *
 * The LSTM cell state decays toward zero across long silences, and denormal
 * operands are slow on several targets. Flushing them is scoped to the call so
 * the library never leaves the caller's floating point mode altered. */

typedef struct {
    unsigned long long saved;
    int active;
} fp_scope;

#if defined(__aarch64__)
#    define FPCR_FZ (1ull << 24)
static inline void fp_scope_enter(fp_scope *s, int enable) {
    s->active = enable;
    if (!enable)
        return;
    unsigned long long r;
    __asm__ __volatile__("mrs %0, fpcr" : "=r"(r));
    s->saved = r;
    if (!(r & FPCR_FZ))
        __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ));
}
static inline void fp_scope_leave(const fp_scope *s) {
    if (s->active)
        __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved));
}
#elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64)
#    include <xmmintrin.h>
/* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring
 * denormals-are-zero bit is left alone: it only affects denormal inputs, and
 * it is absent on the earliest x86-64 parts, where writing it would fault. */
#    define MXCSR_FTZ 0x8000u
static inline void fp_scope_enter(fp_scope *s, int enable) {
    s->active = enable;
    if (!enable)
        return;
    s->saved = _mm_getcsr();
    _mm_setcsr((unsigned)s->saved | MXCSR_FTZ);
}
static inline void fp_scope_leave(const fp_scope *s) {
    if (s->active)
        _mm_setcsr((unsigned)s->saved);
}
#else
static inline void fp_scope_enter(fp_scope *s, int enable) {
    s->saved = 0;
    s->active = enable;
}
static inline void fp_scope_leave(const fp_scope *s) { (void)s; }
#endif

/* ---- pointwise ------------------------------------------------------- */

static inline float leaky_(float x) { return x >= 0.0f ? x : LEAKY_SLOPE * x; }

/* ---- normalisation ----------------------------------------------------
 *
 * InstanceNorm1d over the whole window, one statistic pair per channel. The
 * moments are accumulated in double and the deviations in a second pass: the
 * filterbank output runs to five significant digits over 5325 steps, and a
 * single-pass float variance loses the low ones to cancellation.
 *
 * This is also why the model cannot be run as a stream. Every frame's value
 * depends on statistics of the entire ten-second window. */
#define MAX_CHANNELS C1

static void instance_norm_leaky(float *x, int steps, int ch, const float *scale, const float *shift) {
    double mean[MAX_CHANNELS], inv[MAX_CHANNELS];

    for (int c = 0; c < ch; ++c)
        mean[c] = 0.0;
    for (int t = 0; t < steps; ++t) {
        const float *row = x + (size_t)t * ch;
        for (int c = 0; c < ch; ++c)
            mean[c] += row[c];
    }
    for (int c = 0; c < ch; ++c)
        mean[c] /= steps;

    for (int c = 0; c < ch; ++c)
        inv[c] = 0.0;
    for (int t = 0; t < steps; ++t) {
        const float *row = x + (size_t)t * ch;
        for (int c = 0; c < ch; ++c) {
            const double d = row[c] - mean[c];
            inv[c] += d * d;
        }
    }
    for (int c = 0; c < ch; ++c)
        inv[c] = 1.0 / sqrt(inv[c] / steps + INSTANCE_NORM_EPS);

    for (int t = 0; t < steps; ++t) {
        float *row = x + (size_t)t * ch;
        for (int c = 0; c < ch; ++c)
            row[c] = leaky_((float)((row[c] - mean[c]) * inv[c] * scale[c] + shift[c]));
    }
}

/* ---- convolution stages ----------------------------------------------
 *
 * One call covers a whole stage: convolution, optional magnitude, pooling by
 * three, then the channel statistics. lda is how far apart consecutive
 * output windows start in the source, which is the filterbank's hop for the
 * first stage and one frame for the later ones. */
static void conv_pool_norm(pyseg_t *v,
                           float *dst,
                           const float *src,
                           int lda,
                           const float *weight,
                           const float *bias,
                           int steps,
                           int ch,
                           int k,
                           int take_abs,
                           const float *scale,
                           const float *shift) {
    for (int t0 = 0; t0 < steps; t0 += TILE) {
        const int n = steps - t0 < TILE ? steps - t0 : TILE;
        v->k->gemm_nt(v->tile, ch, src + (size_t)t0 * lda, lda, weight, bias, n, ch, k);

        float *out = dst + (size_t)(t0 / 3) * ch;
        for (int p = 0; p < n / 3; ++p) {
            const float *a = v->tile + (size_t)(3 * p) * ch;
            const float *b = a + ch, *c = b + ch;
            float *o = out + (size_t)p * ch;
            if (take_abs) {
                for (int i = 0; i < ch; ++i) {
                    const float x = fabsf(a[i]), y = fabsf(b[i]), z = fabsf(c[i]);
                    o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z);
                }
            } else {
                for (int i = 0; i < ch; ++i) {
                    const float x = a[i], y = b[i], z = c[i];
                    o[i] = x > y ? (x > z ? x : z) : (y > z ? y : z);
                }
            }
        }
    }
    instance_norm_leaky(dst, steps / 3, ch, scale, shift);
}

/* ---- LSTM -------------------------------------------------------------
 *
 * The input projection of every time step shares one weight matrix, so it runs
 * as a single pass over every step of every window in the batch before the
 * recurrence starts, which is three fifths of the layer's arithmetic done at
 * full matrix-multiply efficiency.
 *
 * What is left is the hidden-to-hidden product, one per step, and it is the
 * reason batching exists. On its own it reads a 256 KB weight matrix to
 * perform 65536 multiply-adds, so it runs at the speed of memory rather than
 * of arithmetic and is the single most expensive part of a window. Windows are
 * independent, so running several of them in step turns that matrix-vector
 * product into a small matrix-matrix product: one weight load then serves
 * every window, and the recurrence speeds up by better than a factor of two at
 * a batch of four. */
static void lstm_direction(pyseg_t *v,
                           int count,
                           const float *in,
                           int lda,
                           int k,
                           const pyseg_lstm_weights *w,
                           float *out,
                           int offset,
                           int reverse) {
    v->k->gemm_nt(v->pre, GATES, in, lda, w->ih, w->bias, count * PYSEG_FRAMES, GATES, k);

    float *h = v->state, *c = v->state + (size_t)v->batch * HID;
    memset(h, 0, (size_t)count * HID * sizeof(float));
    memset(c, 0, (size_t)count * HID * sizeof(float));

    for (int step = 0; step < PYSEG_FRAMES; ++step) {
        const int t = reverse ? PYSEG_FRAMES - 1 - step : step;
        v->k->gemm_nt(v->gates, GATES, h, HID, w->hh, NULL, count, GATES, HID);
        for (int b = 0; b < count; ++b) {
            float *hb = h + (size_t)b * HID;
            v->k->lstm_cell(hb, c + (size_t)b * HID, v->gates + (size_t)b * GATES,
                            v->pre + (size_t)b * PRE_STRIDE + (size_t)t * GATES, HID);
            memcpy(out + (size_t)b * ACT_STRIDE + (size_t)t * BIDIR + offset, hb,
                   HID * sizeof(float));
        }
    }
}

/* ---- forward pass ---------------------------------------------------- */

/* Everything up to the LSTM, for one window of the batch. */
static void front_end(pyseg_t *v, int slot) {
    const pyseg_weights *w = &v->w;

    /* Waveform normalisation: one channel, statistics over the whole window. */
    {
        double mean = 0.0, var = 0.0;
        for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i)
            mean += v->wav[i];
        mean /= PYSEG_WINDOW_SAMPLES;
        for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i) {
            const double d = v->wav[i] - mean;
            var += d * d;
        }
        const double inv = 1.0 / sqrt(var / PYSEG_WINDOW_SAMPLES + INSTANCE_NORM_EPS);
        const float scale = w->wavnorm[0], shift = w->wavnorm[1];
        for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i)
            v->wav[i] = (float)((v->wav[i] - mean) * inv) * scale + shift;
    }

    conv_pool_norm(v, v->pool1, v->wav, SINC_HOP, w->sinc_w, NULL, T1, C1, SINC_K, 1, w->norm_w[0],
                   w->norm_b[0]);
    conv_pool_norm(v, v->pool2, v->pool1, C1, w->conv_w[0], w->conv_b[0], T2, C2, K2, 0, w->norm_w[1],
                   w->norm_b[1]);
    conv_pool_norm(v, v->feat + (size_t)slot * FEAT_STRIDE, v->pool2, C2, w->conv_w[1], w->conv_b[1],
                   T3, C3, K3, 0, w->norm_w[2], w->norm_b[2]);
}

/* The LSTM stack, the feed-forward layers and the softmax, over a whole batch
 * whose features the front end has already produced. */
static void back_end(pyseg_t *v, int count, float *probs) {
    const pyseg_weights *w = &v->w;
    const pyseg_kernels *k = v->k;
    const int rows = count * PYSEG_FRAMES;

    lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][0], v->acta, 0, 0);
    lstm_direction(v, count, v->feat, C3, C3, &w->lstm[0][1], v->acta, HID, 1);
    for (int layer = 1; layer < 4; ++layer) {
        const float *in = (layer & 1) ? v->acta : v->actb;
        float *out = (layer & 1) ? v->actb : v->acta;
        lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][0], out, 0, 0);
        lstm_direction(v, count, in, BIDIR, BIDIR, &w->lstm[layer][1], out, HID, 1);
    }

    /* Layer 3 wrote actb; the feed-forward stack then ping-pongs back. Both
     * layers are narrower than what they read, so writing over the buffer the
     * previous layer occupied is safe. */
    k->gemm_nt(v->acta, HID, v->actb, BIDIR, w->lin_w[0], w->lin_b[0], rows, HID, BIDIR);
    for (int i = 0; i < rows * HID; ++i)
        v->acta[i] = leaky_(v->acta[i]);
    k->gemm_nt(v->actb, HID, v->acta, HID, w->lin_w[1], w->lin_b[1], rows, HID, HID);
    for (int i = 0; i < rows * HID; ++i)
        v->actb[i] = leaky_(v->actb[i]);

    k->gemm_nt(probs, PYSEG_CLASSES, v->actb, HID, w->cls_w, w->cls_b, rows, PYSEG_CLASSES, HID);

    for (int t = 0; t < rows; ++t) {
        float *row = probs + (size_t)t * PYSEG_CLASSES;
        float max = row[0];
        for (int i = 1; i < PYSEG_CLASSES; ++i)
            if (row[i] > max)
                max = row[i];
        float sum = 0.0f;
        for (int i = 0; i < PYSEG_CLASSES; ++i) {
            row[i] = expf(row[i] - max);
            sum += row[i];
        }
        const float inv = 1.0f / sum;
        for (int i = 0; i < PYSEG_CLASSES; ++i)
            row[i] *= inv;
    }
}

/* ---- lifecycle ------------------------------------------------------- */

pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch) {
    if (batch < 1 || batch > PYSEG_MAX_BATCH)
        return NULL;

    pyseg_t *v = calloc(1, sizeof *v);
    if (!v)
        return NULL;
    v->batch = batch;

    if (path) {
        const int verify = !(flags & PYSEG_NO_VERIFY);
        if (pyseg_weights_map(path, verify, &v->w, &v->map, &v->map_len) != 0) {
            free(v);
            return NULL;
        }
    } else {
        size_t len = 0;
        const void *blob = pyseg_weights_embedded(&len);
        if (!blob || pyseg_weights_parse(blob, len, 0, &v->w) != 0) {
            free(v);
            return NULL;
        }
    }

    /* One allocation, carved into 64-byte aligned buffers in this order. */
    const size_t b = (size_t)batch;
    const size_t sizes[] = {
        PYSEG_WINDOW_SAMPLES + 8, /* wav, with slack past the last filter window */
        (size_t)P1 * C1,          /* pool1 */
        (size_t)P2 * C2,          /* pool2 */
        (size_t)TILE * C1,        /* tile */
        b * FEAT_STRIDE,          /* feat */
        b * PRE_STRIDE,           /* pre */
        b * ACT_STRIDE,           /* acta */
        b * ACT_STRIDE,           /* actb */
        b * GATES,                /* gates */
        2 * b * HID,              /* state */
    };
    float **slots[] = {&v->wav, &v->pool1, &v->pool2, &v->tile,  &v->feat,
                       &v->pre, &v->acta,  &v->actb,  &v->gates, &v->state};

    size_t total = 0;
    for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i)
        total += (sizes[i] * sizeof(float) + 63) / 64 * 64;
    if (posix_memalign(&v->arena, 64, total) != 0 || !v->arena) {
        pyseg_weights_unmap(v->map, v->map_len);
        free(v);
        return NULL;
    }
    memset(v->arena, 0, total);
    v->arena_bytes = total;

    unsigned char *p = v->arena;
    for (size_t i = 0; i < sizeof sizes / sizeof *sizes; ++i) {
        *slots[i] = (float *)(void *)p;
        p += (sizes[i] * sizeof(float) + 63) / 64 * 64;
    }

    v->k = (flags & PYSEG_SCALAR_KERNELS) ? pyseg_kernels_scalar() : pyseg_kernels_simd();
    v->ftz = (flags & PYSEG_NO_FTZ) == 0;
    return v;
}

pyseg_t *pyseg_create_ex(const char *path, unsigned flags) {
    return pyseg_create_batch(path, flags, 1);
}

pyseg_t *pyseg_create(void) { return pyseg_create_ex(NULL, 0); }

pyseg_t *pyseg_create_from_file(const char *path) { return pyseg_create_ex(path, 0); }

void pyseg_destroy(pyseg_t *v) {
    if (!v)
        return;
    pyseg_weights_unmap(v->map, v->map_len);
    free(v->arena);
    free(v);
}

const char *pyseg_backend_name(const pyseg_t *v) { return v->k->name; }

int pyseg_batch(const pyseg_t *v) { return v->batch; }

size_t pyseg_memory_bytes(const pyseg_t *v) {
    return sizeof *v + v->arena_bytes + (v->map ? v->map_len : 0);
}

/* ---- entry points ----------------------------------------------------- */

void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs) {
    if (count < 1 || count > v->batch)
        return;
    fp_scope fp;
    fp_scope_enter(&fp, v->ftz);
    for (int b = 0; b < count; ++b) {
        memcpy(v->wav, samples + (size_t)b * PYSEG_WINDOW_SAMPLES,
               PYSEG_WINDOW_SAMPLES * sizeof(float));
        front_end(v, b);
    }
    back_end(v, count, probs);
    fp_scope_leave(&fp);
}

void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs) {
    pyseg_process_batch_f32(v, samples, 1, probs);
}

void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs) {
    fp_scope fp;
    fp_scope_enter(&fp, v->ftz);
    for (int i = 0; i < PYSEG_WINDOW_SAMPLES; ++i)
        v->wav[i] = (float)samples[i] * (1.0f / 32768.0f);
    front_end(v, 0);
    back_end(v, 1, probs);
    fp_scope_leave(&fp);
}

/* ---- powerset decoding -----------------------------------------------
 *
 * The seven classes are, in order, the empty set, the three singletons and the
 * three pairs, which is the order itertools.combinations produces and the
 * order the classifier was trained in. */
static const unsigned char powerset_members[PYSEG_CLASSES] = {
    0x0, 0x1, 0x2, 0x4, 0x3, 0x5, 0x6,
};

void pyseg_to_multilabel(const float *probs, float *active) {
    for (int t = 0; t < PYSEG_FRAMES; ++t) {
        const float *row = probs + (size_t)t * PYSEG_CLASSES;
        int best = 0;
        for (int i = 1; i < PYSEG_CLASSES; ++i)
            if (row[i] > row[best])
                best = i;
        const unsigned mask = powerset_members[best];
        float *out = active + (size_t)t * PYSEG_SPEAKERS;
        for (int s = 0; s < PYSEG_SPEAKERS; ++s)
            out[s] = (mask >> s) & 1u ? 1.0f : 0.0f;
    }
}

int pyseg_powerset_class(int speaker_a, int speaker_b) {
    unsigned mask = 0;
    if (speaker_a >= 0)
        mask |= 1u << speaker_a;
    if (speaker_b >= 0)
        mask |= 1u << speaker_b;
    for (int i = 0; i < PYSEG_CLASSES; ++i)
        if (powerset_members[i] == mask)
            return i;
    return -1;
}