"""Sliding-window aggregation, the reference for src/slide.c. pyannote's Inference cuts a recording into overlapping ten-second windows, converts each window's powerset output to per-speaker activity, and averages the windows with a Hamming taper. The taper lets a frame sitting at the edge of one window, where the LSTM has only seen context on one side, defer to the window that holds it in the middle. Run as a script it checks this implementation against pyannote's own Inference on real audio: python tools/pipeline.py """ import sys import numpy as np from model import ( FRAME_STEP, NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE, checkpoint_path, powerset_mapping, ) TRACK_SPEECH, TRACK_OVERLAP, TRACK_COUNT = 0, 1, 2 NUM_TRACKS = 3 def rint_div(n: int, d: int) -> int: """Round n/d half to even, matching numpy.rint on the same ratio.""" q, r = divmod(n, d) if 2 * r < d: return q if 2 * r > d: return q + 1 return q + (q & 1) def window_starts(num_samples: int, step_samples: int) -> list[int]: if num_samples < NUM_SAMPLES: return [0] whole = (num_samples - NUM_SAMPLES) // step_samples + 1 starts = [i * step_samples for i in range(whole)] if (num_samples - NUM_SAMPLES) % step_samples > 0: starts.append(whole * step_samples) return starts def num_aggregated_frames(num_samples: int, step: float) -> int: step_samples = int(step * SAMPLE_RATE + 0.5) starts = window_starts(num_samples, step_samples) frames = rint_div(NUM_SAMPLES + (len(starts) - 1) * step_samples, FRAME_STEP) + 1 padded = num_samples < NUM_SAMPLES or (num_samples - NUM_SAMPLES) % step_samples > 0 if padded: frames = min(frames, num_samples // FRAME_STEP + 1) return frames def powerset_windows(samples: np.ndarray, step: float, batch: int = 8) -> np.ndarray: """[num_windows, NUM_FRAMES, 7] class probabilities from the torch model.""" import torch from pyannote.audio import Model torch.set_num_threads(1) model = Model.from_pretrained(checkpoint_path()) model.eval() step_samples = int(step * SAMPLE_RATE + 0.5) starts = window_starts(len(samples), step_samples) windows = np.zeros((len(starts), NUM_SAMPLES), dtype=np.float32) for i, s in enumerate(starts): chunk = samples[s : s + NUM_SAMPLES] windows[i, : len(chunk)] = chunk out = [] with torch.no_grad(): for i in range(0, len(windows), batch): logp = model(torch.from_numpy(windows[i : i + batch])[:, None, :]) out.append(torch.exp(logp).numpy()) return np.concatenate(out) def window_tracks(probs: np.ndarray) -> np.ndarray: """[num_windows, NUM_FRAMES, 3] speech / overlap / speaker-count votes. The powerset output is decoded by taking the most likely class, which is the hard decoding pyannote uses, so each window contributes integers rather than probabilities; the averaging across windows is what makes the aggregate continuous. """ mapping = powerset_mapping() active = mapping[probs.argmax(axis=-1)] speakers = active.sum(axis=-1) return np.stack( [(speakers >= 1).astype(np.float32), (speakers >= 2).astype(np.float32), speakers], axis=-1, ) def overlap_add(votes: np.ndarray, num_samples: int, step: float) -> np.ndarray: """Hamming-weighted average of per-window tracks onto the frame grid.""" step_samples = int(step * SAMPLE_RATE + 0.5) starts = window_starts(num_samples, step_samples) frames = num_aggregated_frames(num_samples, step) taper = np.hamming(NUM_FRAMES).astype(np.float32)[:, None] total = np.zeros((frames, votes.shape[-1]), dtype=np.float32) weight = np.zeros((frames, votes.shape[-1]), dtype=np.float32) for i, start in enumerate(starts): base = rint_div(start, FRAME_STEP) n = min(NUM_FRAMES, frames - base) if n <= 0: continue total[base : base + n] += votes[i, :n] * taper[:n] weight[base : base + n] += taper[:n] return np.where(weight > 1e-12, total / np.maximum(weight, 1e-12), 0.0) def aggregate_tracks(samples: np.ndarray, step: float = 1.0) -> np.ndarray: return overlap_add( window_tracks(powerset_windows(samples, step)), len(samples), step ) def main() -> int: """Checks the speech track against pyannote's own VoiceActivityDetection inference, which is the aggregation this file reimplements.""" import soundfile as sf import torch from pyannote.audio import Inference, Model path = sys.argv[1] if len(sys.argv) > 1 else "testdata_aepyx.wav" limit = int(sys.argv[2]) if len(sys.argv) > 2 else 45 * SAMPLE_RATE wav, rate = sf.read(path, dtype="float32", always_2d=True) samples = np.ascontiguousarray(wav[:limit, 0]) if rate != SAMPLE_RATE: raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz") torch.set_num_threads(1) model = Model.from_pretrained(checkpoint_path()) inference = Inference( model, step=1.0, pre_aggregation_hook=lambda s: np.max(s, axis=-1, keepdims=True), ) want = inference( {"waveform": torch.from_numpy(samples)[None, :], "sample_rate": SAMPLE_RATE} ).data[:, 0] got = aggregate_tracks(samples)[:, TRACK_SPEECH] print(f"frames: pyannote {len(want)}, here {len(got)}") n = min(len(want), len(got)) diff = np.abs(want[:n] - got[:n]).max() print(f"max |speech score diff| over {n} frames: {diff:.3e}") ok = len(want) == len(got) and diff <= 1e-5 print("AGGREGATION OK" if ok else "AGGREGATION MISMATCH") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())