/* Sliding-window inference over a whole recording, and binarization. * * The model sees ten seconds at a time and its per-frame statistics depend on * the whole window, so a recording is covered by overlapping windows and their * frame scores are averaged. pyannote weights that average with a Hamming * taper, which lets a frame near a window's edge, where the LSTM has seen * context on one side only, defer to the window that holds it in the middle. * * Windows are independent, so the loop parallelises exactly. Each thread takes * a contiguous run of windows and accumulates into a private plane covering * only the frames that run reaches; the planes of two adjacent threads overlap * by at most one window's worth of frames, and the main thread sums them at * the end. Contiguous runs rather than interleaved ones are what keeps those * planes small: interleaving would give every thread a plane as large as the * whole recording. */ #include "pyseg.h" #include #include #include #ifdef PYSEG_THREADS # include #endif #define AGG_EPSILON 1e-12f /* Four windows per engine call: the point at which sharing each LSTM weight * load stops paying for the registers it costs, measured on both an Apple * M-series core and an x86 server core. */ #define DEFAULT_BATCH 4 void pyseg_slide_params_default(pyseg_slide_params *p) { p->step = 1.0; p->threads = 1; p->batch = DEFAULT_BATCH; p->flags = 0; p->weights = NULL; } void pyseg_binarize_params_default(pyseg_binarize_params *p) { /* The powerset model emits hard per-frame decisions, so an aggregated * score is the fraction of windows that voted yes and the thresholds sit * at one half. pyannote fixes them there for this model rather than * tuning them per corpus. */ p->onset = 0.5; p->offset = 0.5; p->min_duration_on = 0.0; p->min_duration_off = 0.0; } double pyseg_frame_time(size_t frame) { return (double)(frame * PYSEG_FRAME_STEP) / PYSEG_SAMPLE_RATE; } /* Centre of a frame, which is where its score is defined and therefore where * segment boundaries land. */ static double frame_middle(size_t frame) { return pyseg_frame_time(frame) + 0.5 * PYSEG_FRAME_SIZE / PYSEG_SAMPLE_RATE; } /* Rounds n/d to the nearest integer, halves to even, in exact integer * arithmetic. Window starts do not fall on frame boundaries, so which way a * tie goes decides a frame of alignment against the reference. */ static size_t rint_div(size_t n, size_t d) { const size_t q = n / d, r = n % d; if (2 * r < d) return q; if (2 * r > d) return q + 1; return q + (q & 1); } /* Windows the recording is cut into: the complete ones plus, when the tail * does not land on a window boundary, one zero-padded window covering it. */ static size_t window_count(size_t num_samples, size_t step_samples) { if (num_samples < PYSEG_WINDOW_SAMPLES) return 1; const size_t whole = (num_samples - PYSEG_WINDOW_SAMPLES) / step_samples + 1; return whole + ((num_samples - PYSEG_WINDOW_SAMPLES) % step_samples > 0 ? 1 : 0); } static size_t step_samples_of(double step) { const double s = step * PYSEG_SAMPLE_RATE + 0.5; return s < 1.0 ? 1 : (size_t)s; } /* First aggregated frame a window starting at sample contributes to. */ static size_t start_frame_of(size_t sample) { return rint_div(sample, PYSEG_FRAME_STEP); } size_t pyseg_slide_frames(size_t num_samples, double step) { if (num_samples == 0 || step <= 0.0) return 0; const size_t step_samples = step_samples_of(step); const size_t windows = window_count(num_samples, step_samples); size_t frames = rint_div(PYSEG_WINDOW_SAMPLES + (windows - 1) * step_samples, PYSEG_FRAME_STEP) + 1; /* A zero-padded tail window predicts past the end of the recording; those * frames are dropped, as pyannote does after aggregating. */ const int padded = num_samples < PYSEG_WINDOW_SAMPLES || (num_samples - PYSEG_WINDOW_SAMPLES) % step_samples > 0; if (padded) { const size_t limit = num_samples / PYSEG_FRAME_STEP + 1; if (limit < frames) frames = limit; } return frames; } /* ---- window loop ------------------------------------------------------ */ typedef struct { const float *samples; /* one of these two is set */ const int16_t *samples_s16; size_t num_samples; size_t step_samples; size_t frames; const float *taper; /* [PYSEG_FRAMES] */ const char *weights; unsigned flags; int batch; } slide_job; typedef struct { const slide_job *job; size_t first_window, last_window; /* half-open */ size_t first_frame, frame_count; float *sum; /* [frame_count][PYSEG_TRACKS] */ float *count; /* [frame_count][PYSEG_TRACKS] */ int failed; } slide_worker; /* Copies one window out of the recording, zero-filling past the end. */ static void gather(const slide_job *job, size_t index, float *dst) { const size_t start = index * job->step_samples; size_t n = 0; if (start < job->num_samples) { n = job->num_samples - start; if (n > PYSEG_WINDOW_SAMPLES) n = PYSEG_WINDOW_SAMPLES; if (job->samples) memcpy(dst, job->samples + start, n * sizeof(float)); else for (size_t i = 0; i < n; ++i) dst[i] = (float)job->samples_s16[start + i] * (1.0f / 32768.0f); } memset(dst + n, 0, (PYSEG_WINDOW_SAMPLES - n) * sizeof(float)); } /* Adds one window's decoded votes into this thread's plane. */ static void accumulate(slide_worker *self, size_t index, const float *active) { const slide_job *job = self->job; const size_t base = start_frame_of(index * job->step_samples) - self->first_frame; for (int t = 0; t < PYSEG_FRAMES; ++t) { const size_t row = base + (size_t)t; if (row >= self->frame_count) break; const float *a = active + (size_t)t * PYSEG_SPEAKERS; const float speakers = a[0] + a[1] + a[2]; const float weight = job->taper[t]; float *sum = self->sum + row * PYSEG_TRACKS; float *count = self->count + row * PYSEG_TRACKS; sum[PYSEG_TRACK_SPEECH] += (speakers >= 1.0f ? weight : 0.0f); sum[PYSEG_TRACK_OVERLAP] += (speakers >= 2.0f ? weight : 0.0f); sum[PYSEG_TRACK_COUNT] += speakers * weight; count[PYSEG_TRACK_SPEECH] += weight; count[PYSEG_TRACK_OVERLAP] += weight; count[PYSEG_TRACK_COUNT] += weight; } } static void *worker_run(void *arg) { slide_worker *self = arg; const slide_job *job = self->job; const int batch = job->batch; pyseg_t *engine = pyseg_create_batch(job->weights, job->flags, batch); float *windows = malloc((size_t)batch * PYSEG_WINDOW_SAMPLES * sizeof(float)); float *probs = malloc((size_t)batch * PYSEG_FRAMES * PYSEG_CLASSES * sizeof(float)); float *active = malloc((size_t)PYSEG_FRAMES * PYSEG_SPEAKERS * sizeof(float)); if (!engine || !windows || !probs || !active) { self->failed = 1; goto done; } for (size_t index = self->first_window; index < self->last_window;) { int count = 0; while (count < batch && index + (size_t)count < self->last_window) { gather(job, index + (size_t)count, windows + (size_t)count * PYSEG_WINDOW_SAMPLES); ++count; } pyseg_process_batch_f32(engine, windows, count, probs); for (int b = 0; b < count; ++b) { pyseg_to_multilabel(probs + (size_t)b * PYSEG_FRAMES * PYSEG_CLASSES, active); accumulate(self, index + (size_t)b, active); } index += (size_t)count; } done: free(active); free(probs); free(windows); pyseg_destroy(engine); return NULL; } static size_t slide_run(const float *samples, const int16_t *samples_s16, size_t num_samples, const pyseg_slide_params *params, float *tracks) { pyseg_slide_params defaults; if (!params) { pyseg_slide_params_default(&defaults); params = &defaults; } if ((!samples && !samples_s16) || num_samples == 0 || !tracks || params->step <= 0.0) return 0; const size_t step_samples = step_samples_of(params->step); if (step_samples > PYSEG_WINDOW_SAMPLES || params->batch > PYSEG_MAX_BATCH) return 0; const size_t windows = window_count(num_samples, step_samples); const size_t frames = pyseg_slide_frames(num_samples, params->step); if (frames == 0) return 0; /* np.hamming over the window's frames. It never reaches zero, so every * frame keeps a vote however close to the edge it sits. */ float taper[PYSEG_FRAMES]; for (int t = 0; t < PYSEG_FRAMES; ++t) taper[t] = (float)(0.54 - 0.46 * cos(2.0 * M_PI * t / (PYSEG_FRAMES - 1))); const slide_job job = { .samples = samples, .samples_s16 = samples_s16, .num_samples = num_samples, .step_samples = step_samples, .frames = frames, .taper = taper, .weights = params->weights, .flags = params->flags, .batch = params->batch > 0 ? params->batch : DEFAULT_BATCH, }; int nthreads = params->threads > 0 ? params->threads : 1; if (nthreads > PYSEG_MAX_THREADS) nthreads = PYSEG_MAX_THREADS; #ifndef PYSEG_THREADS nthreads = 1; #endif if ((size_t)nthreads > windows) nthreads = (int)windows; slide_worker *workers = calloc((size_t)nthreads, sizeof *workers); if (!workers) return 0; int failed = 0; for (int i = 0; i < nthreads; ++i) { slide_worker *w = &workers[i]; w->job = &job; w->first_window = windows * (size_t)i / (size_t)nthreads; w->last_window = windows * (size_t)(i + 1) / (size_t)nthreads; w->first_frame = start_frame_of(w->first_window * step_samples); size_t end = start_frame_of((w->last_window - 1) * step_samples) + PYSEG_FRAMES; if (end > frames) end = frames; w->frame_count = end > w->first_frame ? end - w->first_frame : 0; w->sum = calloc(w->frame_count * PYSEG_TRACKS + 1, sizeof(float)); w->count = calloc(w->frame_count * PYSEG_TRACKS + 1, sizeof(float)); if (!w->sum || !w->count) failed = 1; } #ifdef PYSEG_THREADS pthread_t *threads = nthreads > 1 ? calloc((size_t)nthreads - 1, sizeof *threads) : NULL; int spawned = 0; if (!failed && nthreads > 1 && !threads) failed = 1; if (!failed) for (int i = 1; i < nthreads; ++i) { if (pthread_create(&threads[spawned], NULL, worker_run, &workers[i]) != 0) { failed = 1; break; } ++spawned; } #endif if (!failed) worker_run(&workers[0]); #ifdef PYSEG_THREADS for (int i = 0; i < spawned; ++i) pthread_join(threads[i], NULL); free(threads); #endif for (int i = 0; i < nthreads; ++i) failed |= workers[i].failed; if (!failed) { memset(tracks, 0, frames * PYSEG_TRACKS * sizeof(float)); float *count = calloc(frames * PYSEG_TRACKS, sizeof(float)); if (!count) { failed = 1; } else { for (int i = 0; i < nthreads; ++i) { const slide_worker *w = &workers[i]; const size_t base = w->first_frame * PYSEG_TRACKS; for (size_t j = 0; j < w->frame_count * PYSEG_TRACKS; ++j) { tracks[base + j] += w->sum[j]; count[base + j] += w->count[j]; } } for (size_t j = 0; j < frames * PYSEG_TRACKS; ++j) tracks[j] = count[j] > AGG_EPSILON ? tracks[j] / count[j] : 0.0f; free(count); } } for (int i = 0; i < nthreads; ++i) { free(workers[i].sum); free(workers[i].count); } free(workers); return failed ? 0 : frames; } size_t pyseg_slide(const float *samples, size_t num_samples, const pyseg_slide_params *params, float *tracks) { return slide_run(samples, NULL, num_samples, params, tracks); } size_t pyseg_slide_s16(const int16_t *samples, size_t num_samples, const pyseg_slide_params *params, float *tracks) { return slide_run(NULL, samples, num_samples, params, tracks); } /* ---- binarization ----------------------------------------------------- * * Hysteresis over the aggregated score: a region opens when the score rises * above onset and closes when it falls below offset, so a score hovering * at the threshold does not chatter. The first frame decides the initial state * on its own; every later frame is a possible transition, and a transition is * timestamped at the frame that caused it. */ size_t pyseg_binarize(const float *scores, size_t num_frames, int track, const pyseg_binarize_params *params, pyseg_segment *out, size_t out_cap) { pyseg_binarize_params defaults; if (!params) { pyseg_binarize_params_default(&defaults); params = &defaults; } if (!scores || num_frames == 0 || track < 0 || track >= PYSEG_TRACKS) return 0; const size_t cap = out ? out_cap : 0; size_t n = 0; int is_active = scores[(size_t)track] > params->onset; double start = frame_middle(0); double time = start; for (size_t f = 1; f < num_frames; ++f) { const double value = scores[f * PYSEG_TRACKS + (size_t)track]; time = frame_middle(f); if (is_active) { if (value < params->offset) { if (n < cap) { out[n].start = start; out[n].end = time; } ++n; is_active = 0; } } else if (value > params->onset) { start = time; is_active = 1; } } if (is_active) { if (n < cap) { out[n].start = start; out[n].end = time; } ++n; } if (!out || n > cap) return n; /* Both passes look across neighbours, so they run over the collected list * rather than during the scan: a gap is only short once both of the * segments around it are known. */ if (params->min_duration_off > 0.0) { size_t w = 0; for (size_t i = 0; i < n; ++i) { if (w > 0 && out[i].start - out[w - 1].end < params->min_duration_off) out[w - 1].end = out[i].end; else out[w++] = out[i]; } n = w; } if (params->min_duration_on > 0.0) { size_t w = 0; for (size_t i = 0; i < n; ++i) if (out[i].end - out[i].start >= params->min_duration_on) out[w++] = out[i]; n = w; } return n; }