diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:03:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:13:28 +0200 |
| commit | 64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch) | |
| tree | 85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /tools/pipeline.py | |
| download | pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip | |
Init
Diffstat (limited to 'tools/pipeline.py')
| -rw-r--r-- | tools/pipeline.py | 169 |
1 files changed, 169 insertions, 0 deletions
diff --git a/tools/pipeline.py b/tools/pipeline.py new file mode 100644 index 0000000..c4f675d --- /dev/null +++ b/tools/pipeline.py @@ -0,0 +1,169 @@ +"""Sliding-window aggregation, the reference for src/slide.c. + +pyannote's Inference cuts a recording into overlapping ten-second windows, +converts each window's powerset output to per-speaker activity, and averages +the windows with a Hamming taper. The taper lets a frame sitting at the edge of +one window, where the LSTM has only seen context on one side, defer to the +window that holds it in the middle. + +Run as a script it checks this implementation against pyannote's own Inference +on real audio: + + python tools/pipeline.py <audio.wav> +""" + +import sys + +import numpy as np + +from model import ( + FRAME_STEP, + NUM_FRAMES, + NUM_SAMPLES, + SAMPLE_RATE, + checkpoint_path, + powerset_mapping, +) + +TRACK_SPEECH, TRACK_OVERLAP, TRACK_COUNT = 0, 1, 2 +NUM_TRACKS = 3 + + +def rint_div(n: int, d: int) -> int: + """Round n/d half to even, matching numpy.rint on the same ratio.""" + q, r = divmod(n, d) + if 2 * r < d: + return q + if 2 * r > d: + return q + 1 + return q + (q & 1) + + +def window_starts(num_samples: int, step_samples: int) -> list[int]: + if num_samples < NUM_SAMPLES: + return [0] + whole = (num_samples - NUM_SAMPLES) // step_samples + 1 + starts = [i * step_samples for i in range(whole)] + if (num_samples - NUM_SAMPLES) % step_samples > 0: + starts.append(whole * step_samples) + return starts + + +def num_aggregated_frames(num_samples: int, step: float) -> int: + step_samples = int(step * SAMPLE_RATE + 0.5) + starts = window_starts(num_samples, step_samples) + frames = rint_div(NUM_SAMPLES + (len(starts) - 1) * step_samples, FRAME_STEP) + 1 + padded = num_samples < NUM_SAMPLES or (num_samples - NUM_SAMPLES) % step_samples > 0 + if padded: + frames = min(frames, num_samples // FRAME_STEP + 1) + return frames + + +def powerset_windows(samples: np.ndarray, step: float, batch: int = 8) -> np.ndarray: + """[num_windows, NUM_FRAMES, 7] class probabilities from the torch model.""" + import torch + from pyannote.audio import Model + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + model.eval() + + step_samples = int(step * SAMPLE_RATE + 0.5) + starts = window_starts(len(samples), step_samples) + windows = np.zeros((len(starts), NUM_SAMPLES), dtype=np.float32) + for i, s in enumerate(starts): + chunk = samples[s : s + NUM_SAMPLES] + windows[i, : len(chunk)] = chunk + + out = [] + with torch.no_grad(): + for i in range(0, len(windows), batch): + logp = model(torch.from_numpy(windows[i : i + batch])[:, None, :]) + out.append(torch.exp(logp).numpy()) + return np.concatenate(out) + + +def window_tracks(probs: np.ndarray) -> np.ndarray: + """[num_windows, NUM_FRAMES, 3] speech / overlap / speaker-count votes. + + The powerset output is decoded by taking the most likely class, which is + the hard decoding pyannote uses, so each window contributes integers rather + than probabilities; the averaging across windows is what makes the + aggregate continuous. + """ + mapping = powerset_mapping() + active = mapping[probs.argmax(axis=-1)] + speakers = active.sum(axis=-1) + return np.stack( + [(speakers >= 1).astype(np.float32), (speakers >= 2).astype(np.float32), speakers], + axis=-1, + ) + + +def overlap_add(votes: np.ndarray, num_samples: int, step: float) -> np.ndarray: + """Hamming-weighted average of per-window tracks onto the frame grid.""" + step_samples = int(step * SAMPLE_RATE + 0.5) + starts = window_starts(num_samples, step_samples) + frames = num_aggregated_frames(num_samples, step) + + taper = np.hamming(NUM_FRAMES).astype(np.float32)[:, None] + total = np.zeros((frames, votes.shape[-1]), dtype=np.float32) + weight = np.zeros((frames, votes.shape[-1]), dtype=np.float32) + + for i, start in enumerate(starts): + base = rint_div(start, FRAME_STEP) + n = min(NUM_FRAMES, frames - base) + if n <= 0: + continue + total[base : base + n] += votes[i, :n] * taper[:n] + weight[base : base + n] += taper[:n] + + return np.where(weight > 1e-12, total / np.maximum(weight, 1e-12), 0.0) + + +def aggregate_tracks(samples: np.ndarray, step: float = 1.0) -> np.ndarray: + return overlap_add( + window_tracks(powerset_windows(samples, step)), len(samples), step + ) + + +def main() -> int: + """Checks the speech track against pyannote's own VoiceActivityDetection + inference, which is the aggregation this file reimplements.""" + import soundfile as sf + import torch + from pyannote.audio import Inference, Model + + path = sys.argv[1] if len(sys.argv) > 1 else "testdata_aepyx.wav" + limit = int(sys.argv[2]) if len(sys.argv) > 2 else 45 * SAMPLE_RATE + + wav, rate = sf.read(path, dtype="float32", always_2d=True) + samples = np.ascontiguousarray(wav[:limit, 0]) + if rate != SAMPLE_RATE: + raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz") + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + inference = Inference( + model, + step=1.0, + pre_aggregation_hook=lambda s: np.max(s, axis=-1, keepdims=True), + ) + want = inference( + {"waveform": torch.from_numpy(samples)[None, :], "sample_rate": SAMPLE_RATE} + ).data[:, 0] + + got = aggregate_tracks(samples)[:, TRACK_SPEECH] + + print(f"frames: pyannote {len(want)}, here {len(got)}") + n = min(len(want), len(got)) + diff = np.abs(want[:n] - got[:n]).max() + print(f"max |speech score diff| over {n} frames: {diff:.3e}") + + ok = len(want) == len(got) and diff <= 1e-5 + print("AGGREGATION OK" if ok else "AGGREGATION MISMATCH") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) |
