diff options
Diffstat (limited to 'tools/verify_cli.py')
| -rw-r--r-- | tools/verify_cli.py | 132 |
1 files changed, 132 insertions, 0 deletions
diff --git a/tools/verify_cli.py b/tools/verify_cli.py new file mode 100644 index 0000000..15e7cef --- /dev/null +++ b/tools/verify_cli.py @@ -0,0 +1,132 @@ +"""End-to-end check of the built binary against pyannote's own pipelines. + +The parity tests compare tensors; this compares what a user actually gets. It +derives speech and overlapped-speech segments from pyannote for a recording, +runs pyseg_cli over the same file, and reports both the segment boundaries +and the fraction of the timeline the two disagree about, which is the number +that matters for anything downstream. + + python tools/verify_cli.py <audio.wav> [--limit SECONDS] [--step SECONDS] +""" + +import argparse +import subprocess +import sys + +import numpy as np + +from model import SAMPLE_RATE, checkpoint_path + +BINARY = "./pyseg_cli" + + +def run_cli(path: str, mode: str, step: float) -> list[tuple[float, float]]: + out = subprocess.run( + [BINARY, "--mode", mode, "--step", str(step), path], + check=True, + capture_output=True, + text=True, + ).stdout + return [ + (float(a), float(b)) + for a, b in (line.split() for line in out.splitlines() if line.strip()) + ] + + +def run_pyannote(path: str, mode: str, step: float): + """Reference segments for one track. + + Speech goes through pyannote's own VoiceActivityDetection pipeline. Overlap + is assembled here from the same parts, because the pipeline that used to + wrap them was dropped from pyannote.audio: take the second largest + per-speaker score of each frame, which for this model's hard multi-label + output is one exactly when at least two speakers are active, aggregate, and + binarize with the same thresholds. + """ + import torch + from pyannote.audio import Inference, Model + from pyannote.audio.pipelines import VoiceActivityDetection + from pyannote.audio.utils.signal import Binarize + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + + if mode == "speech": + pipeline = VoiceActivityDetection(segmentation=model, step=step) + pipeline.instantiate({"min_duration_on": 0.0, "min_duration_off": 0.0}) + annotation = pipeline(path) + else: + inference = Inference( + model, + step=step, + pre_aggregation_hook=lambda s: np.partition(s, -2, axis=-1)[:, :, -2, np.newaxis], + ) + binarize = Binarize( + onset=0.5, offset=0.5, min_duration_on=0.0, min_duration_off=0.0 + ) + annotation = binarize(inference(path)) + + return [(s.start, s.end) for s in annotation.get_timeline()] + + +def timeline_mask(segments, duration: float, resolution: float = 0.01) -> np.ndarray: + n = int(duration / resolution) + 1 + mask = np.zeros(n, dtype=bool) + for start, end in segments: + mask[int(start / resolution) : int(end / resolution) + 1] = True + return mask + + +def compare(name: str, got, want, duration: float) -> bool: + a, b = timeline_mask(got, duration), timeline_mask(want, duration) + n = min(len(a), len(b)) + disagree = np.count_nonzero(a[:n] != b[:n]) / n + covered = np.count_nonzero(b[:n]) / n + + print(f"{name}: {len(got)} segments here, {len(want)} from pyannote") + print(f" reference covers {covered:6.2%} of the timeline") + print(f" disagreement {disagree:6.4%} of the timeline") + + worst = 0.0 + for (s0, e0), (s1, e1) in zip(got, want): + worst = max(worst, abs(s0 - s1), abs(e0 - e1)) + if len(got) == len(want): + print(f" worst boundary shift {worst * 1e3:.1f} ms") + + ok = disagree <= 1e-3 + print(f" {'OK' if ok else 'MISMATCH'}") + return ok + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("audio") + ap.add_argument("--limit", type=int, default=None, help="seconds of audio to use") + ap.add_argument("--step", type=float, default=1.0) + args = ap.parse_args() + + import soundfile as sf + + info = sf.info(args.audio) + if info.samplerate != SAMPLE_RATE: + raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz") + duration = info.duration if args.limit is None else min(info.duration, args.limit) + + path = args.audio + if args.limit is not None: + wav, rate = sf.read(args.audio, dtype="float32", always_2d=True) + path = "build/verify_clip.wav" + sf.write(path, wav[: args.limit * rate, :1], rate, subtype="PCM_16") + + ok = True + for mode in ("speech", "overlap"): + got = run_cli(path, mode, args.step) + want = run_pyannote(path, mode, args.step) + ok &= compare(mode, got, want, duration) + + print("VERIFY OK" if ok else "VERIFY FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) |
