aboutsummaryrefslogtreecommitdiff
path: root/src/slide.c
blob: be23ec4cbafee5631752eed1562dbbc6f8262e9a (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
/* Sliding-window inference over a whole recording, and binarization.
 *
 * The model sees ten seconds at a time and its per-frame statistics depend on
 * the whole window, so a recording is covered by overlapping windows and their
 * frame scores are averaged. pyannote weights that average with a Hamming
 * taper, which lets a frame near a window's edge, where the LSTM has seen
 * context on one side only, defer to the window that holds it in the middle.
 *
 * Windows are independent, so the loop parallelises exactly. Each thread takes
 * a contiguous run of windows and accumulates into a private plane covering
 * only the frames that run reaches; the planes of two adjacent threads overlap
 * by at most one window's worth of frames, and the main thread sums them at
 * the end. Contiguous runs rather than interleaved ones are what keeps those
 * planes small: interleaving would give every thread a plane as large as the
 * whole recording. */

#include "pyseg.h"

#include <math.h>
#include <stdlib.h>
#include <string.h>

#ifdef PYSEG_THREADS
#    include <pthread.h>
#endif

#define AGG_EPSILON 1e-12f

/* Four windows per engine call: the point at which sharing each LSTM weight
 * load stops paying for the registers it costs, measured on both an Apple
 * M-series core and an x86 server core. */
#define DEFAULT_BATCH 4

void pyseg_slide_params_default(pyseg_slide_params *p) {
    p->step = 1.0;
    p->threads = 1;
    p->batch = DEFAULT_BATCH;
    p->flags = 0;
    p->weights = NULL;
}

void pyseg_binarize_params_default(pyseg_binarize_params *p) {
    /* The powerset model emits hard per-frame decisions, so an aggregated
     * score is the fraction of windows that voted yes and the thresholds sit
     * at one half. pyannote fixes them there for this model rather than
     * tuning them per corpus. */
    p->onset = 0.5;
    p->offset = 0.5;
    p->min_duration_on = 0.0;
    p->min_duration_off = 0.0;
}

double pyseg_frame_time(size_t frame) {
    return (double)(frame * PYSEG_FRAME_STEP) / PYSEG_SAMPLE_RATE;
}

/* Centre of a frame, which is where its score is defined and therefore where
 * segment boundaries land. */
static double frame_middle(size_t frame) {
    return pyseg_frame_time(frame) + 0.5 * PYSEG_FRAME_SIZE / PYSEG_SAMPLE_RATE;
}

/* Rounds n/d to the nearest integer, halves to even, in exact integer
 * arithmetic. Window starts do not fall on frame boundaries, so which way a
 * tie goes decides a frame of alignment against the reference. */
static size_t rint_div(size_t n, size_t d) {
    const size_t q = n / d, r = n % d;
    if (2 * r < d)
        return q;
    if (2 * r > d)
        return q + 1;
    return q + (q & 1);
}

/* Windows the recording is cut into: the complete ones plus, when the tail
 * does not land on a window boundary, one zero-padded window covering it. */
static size_t window_count(size_t num_samples, size_t step_samples) {
    if (num_samples < PYSEG_WINDOW_SAMPLES)
        return 1;
    const size_t whole = (num_samples - PYSEG_WINDOW_SAMPLES) / step_samples + 1;
    return whole + ((num_samples - PYSEG_WINDOW_SAMPLES) % step_samples > 0 ? 1 : 0);
}

static size_t step_samples_of(double step) {
    const double s = step * PYSEG_SAMPLE_RATE + 0.5;
    return s < 1.0 ? 1 : (size_t)s;
}

/* First aggregated frame a window starting at sample contributes to. */
static size_t start_frame_of(size_t sample) { return rint_div(sample, PYSEG_FRAME_STEP); }

size_t pyseg_slide_frames(size_t num_samples, double step) {
    if (num_samples == 0 || step <= 0.0)
        return 0;
    const size_t step_samples = step_samples_of(step);
    const size_t windows = window_count(num_samples, step_samples);
    size_t frames =
        rint_div(PYSEG_WINDOW_SAMPLES + (windows - 1) * step_samples, PYSEG_FRAME_STEP) + 1;

    /* A zero-padded tail window predicts past the end of the recording; those
     * frames are dropped, as pyannote does after aggregating. */
    const int padded =
        num_samples < PYSEG_WINDOW_SAMPLES || (num_samples - PYSEG_WINDOW_SAMPLES) % step_samples > 0;
    if (padded) {
        const size_t limit = num_samples / PYSEG_FRAME_STEP + 1;
        if (limit < frames)
            frames = limit;
    }
    return frames;
}

/* ---- window loop ------------------------------------------------------ */

typedef struct {
    const float *samples; /* one of these two is set */
    const int16_t *samples_s16;
    size_t num_samples;
    size_t step_samples;
    size_t frames;
    const float *taper; /* [PYSEG_FRAMES] */
    const char *weights;
    unsigned flags;
    int batch;
} slide_job;

typedef struct {
    const slide_job *job;
    size_t first_window, last_window; /* half-open */
    size_t first_frame, frame_count;
    float *sum;   /* [frame_count][PYSEG_TRACKS] */
    float *count; /* [frame_count][PYSEG_TRACKS] */
    int failed;
} slide_worker;

/* Copies one window out of the recording, zero-filling past the end. */
static void gather(const slide_job *job, size_t index, float *dst) {
    const size_t start = index * job->step_samples;
    size_t n = 0;
    if (start < job->num_samples) {
        n = job->num_samples - start;
        if (n > PYSEG_WINDOW_SAMPLES)
            n = PYSEG_WINDOW_SAMPLES;
        if (job->samples)
            memcpy(dst, job->samples + start, n * sizeof(float));
        else
            for (size_t i = 0; i < n; ++i)
                dst[i] = (float)job->samples_s16[start + i] * (1.0f / 32768.0f);
    }
    memset(dst + n, 0, (PYSEG_WINDOW_SAMPLES - n) * sizeof(float));
}

/* Adds one window's decoded votes into this thread's plane. */
static void accumulate(slide_worker *self, size_t index, const float *active) {
    const slide_job *job = self->job;
    const size_t base = start_frame_of(index * job->step_samples) - self->first_frame;
    for (int t = 0; t < PYSEG_FRAMES; ++t) {
        const size_t row = base + (size_t)t;
        if (row >= self->frame_count)
            break;
        const float *a = active + (size_t)t * PYSEG_SPEAKERS;
        const float speakers = a[0] + a[1] + a[2];
        const float weight = job->taper[t];
        float *sum = self->sum + row * PYSEG_TRACKS;
        float *count = self->count + row * PYSEG_TRACKS;
        sum[PYSEG_TRACK_SPEECH] += (speakers >= 1.0f ? weight : 0.0f);
        sum[PYSEG_TRACK_OVERLAP] += (speakers >= 2.0f ? weight : 0.0f);
        sum[PYSEG_TRACK_COUNT] += speakers * weight;
        count[PYSEG_TRACK_SPEECH] += weight;
        count[PYSEG_TRACK_OVERLAP] += weight;
        count[PYSEG_TRACK_COUNT] += weight;
    }
}

static void *worker_run(void *arg) {
    slide_worker *self = arg;
    const slide_job *job = self->job;
    const int batch = job->batch;

    pyseg_t *engine = pyseg_create_batch(job->weights, job->flags, batch);
    float *windows = malloc((size_t)batch * PYSEG_WINDOW_SAMPLES * sizeof(float));
    float *probs = malloc((size_t)batch * PYSEG_FRAMES * PYSEG_CLASSES * sizeof(float));
    float *active = malloc((size_t)PYSEG_FRAMES * PYSEG_SPEAKERS * sizeof(float));
    if (!engine || !windows || !probs || !active) {
        self->failed = 1;
        goto done;
    }

    for (size_t index = self->first_window; index < self->last_window;) {
        int count = 0;
        while (count < batch && index + (size_t)count < self->last_window) {
            gather(job, index + (size_t)count, windows + (size_t)count * PYSEG_WINDOW_SAMPLES);
            ++count;
        }
        pyseg_process_batch_f32(engine, windows, count, probs);
        for (int b = 0; b < count; ++b) {
            pyseg_to_multilabel(probs + (size_t)b * PYSEG_FRAMES * PYSEG_CLASSES, active);
            accumulate(self, index + (size_t)b, active);
        }
        index += (size_t)count;
    }

done:
    free(active);
    free(probs);
    free(windows);
    pyseg_destroy(engine);
    return NULL;
}

static size_t slide_run(const float *samples,
                        const int16_t *samples_s16,
                        size_t num_samples,
                        const pyseg_slide_params *params,
                        float *tracks) {
    pyseg_slide_params defaults;
    if (!params) {
        pyseg_slide_params_default(&defaults);
        params = &defaults;
    }
    if ((!samples && !samples_s16) || num_samples == 0 || !tracks || params->step <= 0.0)
        return 0;

    const size_t step_samples = step_samples_of(params->step);
    if (step_samples > PYSEG_WINDOW_SAMPLES || params->batch > PYSEG_MAX_BATCH)
        return 0;

    const size_t windows = window_count(num_samples, step_samples);
    const size_t frames = pyseg_slide_frames(num_samples, params->step);
    if (frames == 0)
        return 0;

    /* np.hamming over the window's frames. It never reaches zero, so every
     * frame keeps a vote however close to the edge it sits. */
    float taper[PYSEG_FRAMES];
    for (int t = 0; t < PYSEG_FRAMES; ++t)
        taper[t] = (float)(0.54 - 0.46 * cos(2.0 * M_PI * t / (PYSEG_FRAMES - 1)));

    const slide_job job = {
        .samples = samples,
        .samples_s16 = samples_s16,
        .num_samples = num_samples,
        .step_samples = step_samples,
        .frames = frames,
        .taper = taper,
        .weights = params->weights,
        .flags = params->flags,
        .batch = params->batch > 0 ? params->batch : DEFAULT_BATCH,
    };

    int nthreads = params->threads > 0 ? params->threads : 1;
    if (nthreads > PYSEG_MAX_THREADS)
        nthreads = PYSEG_MAX_THREADS;
#ifndef PYSEG_THREADS
    nthreads = 1;
#endif
    if ((size_t)nthreads > windows)
        nthreads = (int)windows;

    slide_worker *workers = calloc((size_t)nthreads, sizeof *workers);
    if (!workers)
        return 0;

    int failed = 0;
    for (int i = 0; i < nthreads; ++i) {
        slide_worker *w = &workers[i];
        w->job = &job;
        w->first_window = windows * (size_t)i / (size_t)nthreads;
        w->last_window = windows * (size_t)(i + 1) / (size_t)nthreads;
        w->first_frame = start_frame_of(w->first_window * step_samples);
        size_t end = start_frame_of((w->last_window - 1) * step_samples) + PYSEG_FRAMES;
        if (end > frames)
            end = frames;
        w->frame_count = end > w->first_frame ? end - w->first_frame : 0;
        w->sum = calloc(w->frame_count * PYSEG_TRACKS + 1, sizeof(float));
        w->count = calloc(w->frame_count * PYSEG_TRACKS + 1, sizeof(float));
        if (!w->sum || !w->count)
            failed = 1;
    }

#ifdef PYSEG_THREADS
    pthread_t *threads = nthreads > 1 ? calloc((size_t)nthreads - 1, sizeof *threads) : NULL;
    int spawned = 0;
    if (!failed && nthreads > 1 && !threads)
        failed = 1;
    if (!failed)
        for (int i = 1; i < nthreads; ++i) {
            if (pthread_create(&threads[spawned], NULL, worker_run, &workers[i]) != 0) {
                failed = 1;
                break;
            }
            ++spawned;
        }
#endif
    if (!failed)
        worker_run(&workers[0]);
#ifdef PYSEG_THREADS
    for (int i = 0; i < spawned; ++i)
        pthread_join(threads[i], NULL);
    free(threads);
#endif

    for (int i = 0; i < nthreads; ++i)
        failed |= workers[i].failed;

    if (!failed) {
        memset(tracks, 0, frames * PYSEG_TRACKS * sizeof(float));
        float *count = calloc(frames * PYSEG_TRACKS, sizeof(float));
        if (!count) {
            failed = 1;
        } else {
            for (int i = 0; i < nthreads; ++i) {
                const slide_worker *w = &workers[i];
                const size_t base = w->first_frame * PYSEG_TRACKS;
                for (size_t j = 0; j < w->frame_count * PYSEG_TRACKS; ++j) {
                    tracks[base + j] += w->sum[j];
                    count[base + j] += w->count[j];
                }
            }
            for (size_t j = 0; j < frames * PYSEG_TRACKS; ++j)
                tracks[j] = count[j] > AGG_EPSILON ? tracks[j] / count[j] : 0.0f;
            free(count);
        }
    }

    for (int i = 0; i < nthreads; ++i) {
        free(workers[i].sum);
        free(workers[i].count);
    }
    free(workers);
    return failed ? 0 : frames;
}

size_t pyseg_slide(const float *samples,
                   size_t num_samples,
                   const pyseg_slide_params *params,
                   float *tracks) {
    return slide_run(samples, NULL, num_samples, params, tracks);
}

size_t pyseg_slide_s16(const int16_t *samples,
                       size_t num_samples,
                       const pyseg_slide_params *params,
                       float *tracks) {
    return slide_run(NULL, samples, num_samples, params, tracks);
}

/* ---- binarization -----------------------------------------------------
 *
 * Hysteresis over the aggregated score: a region opens when the score rises
 * above onset and closes when it falls below offset, so a score hovering
 * at the threshold does not chatter. The first frame decides the initial state
 * on its own; every later frame is a possible transition, and a transition is
 * timestamped at the frame that caused it. */
size_t pyseg_binarize(const float *scores,
                      size_t num_frames,
                      int track,
                      const pyseg_binarize_params *params,
                      pyseg_segment *out,
                      size_t out_cap) {
    pyseg_binarize_params defaults;
    if (!params) {
        pyseg_binarize_params_default(&defaults);
        params = &defaults;
    }
    if (!scores || num_frames == 0 || track < 0 || track >= PYSEG_TRACKS)
        return 0;

    const size_t cap = out ? out_cap : 0;
    size_t n = 0;
    int is_active = scores[(size_t)track] > params->onset;
    double start = frame_middle(0);
    double time = start;

    for (size_t f = 1; f < num_frames; ++f) {
        const double value = scores[f * PYSEG_TRACKS + (size_t)track];
        time = frame_middle(f);
        if (is_active) {
            if (value < params->offset) {
                if (n < cap) {
                    out[n].start = start;
                    out[n].end = time;
                }
                ++n;
                is_active = 0;
            }
        } else if (value > params->onset) {
            start = time;
            is_active = 1;
        }
    }
    if (is_active) {
        if (n < cap) {
            out[n].start = start;
            out[n].end = time;
        }
        ++n;
    }

    if (!out || n > cap)
        return n;

    /* Both passes look across neighbours, so they run over the collected list
     * rather than during the scan: a gap is only short once both of the
     * segments around it are known. */
    if (params->min_duration_off > 0.0) {
        size_t w = 0;
        for (size_t i = 0; i < n; ++i) {
            if (w > 0 && out[i].start - out[w - 1].end < params->min_duration_off)
                out[w - 1].end = out[i].end;
            else
                out[w++] = out[i];
        }
        n = w;
    }
    if (params->min_duration_on > 0.0) {
        size_t w = 0;
        for (size_t i = 0; i < n; ++i)
            if (out[i].end - out[i].start >= params->min_duration_on)
                out[w++] = out[i];
        n = w;
    }
    return n;
}