"""End-to-end check of the built binary against pyannote's own pipelines. The parity tests compare tensors; this compares what a user actually gets. It derives speech and overlapped-speech segments from pyannote for a recording, runs pyseg_cli over the same file, and reports both the segment boundaries and the fraction of the timeline the two disagree about, which is the number that matters for anything downstream. python tools/verify_cli.py [--limit SECONDS] [--step SECONDS] """ import argparse import subprocess import sys import numpy as np from model import SAMPLE_RATE, checkpoint_path BINARY = "./pyseg_cli" def run_cli(path: str, mode: str, step: float) -> list[tuple[float, float]]: out = subprocess.run( [BINARY, "--mode", mode, "--step", str(step), path], check=True, capture_output=True, text=True, ).stdout return [ (float(a), float(b)) for a, b in (line.split() for line in out.splitlines() if line.strip()) ] def run_pyannote(path: str, mode: str, step: float): """Reference segments for one track. Speech goes through pyannote's own VoiceActivityDetection pipeline. Overlap is assembled here from the same parts, because the pipeline that used to wrap them was dropped from pyannote.audio: take the second largest per-speaker score of each frame, which for this model's hard multi-label output is one exactly when at least two speakers are active, aggregate, and binarize with the same thresholds. """ import torch from pyannote.audio import Inference, Model from pyannote.audio.pipelines import VoiceActivityDetection from pyannote.audio.utils.signal import Binarize torch.set_num_threads(1) model = Model.from_pretrained(checkpoint_path()) if mode == "speech": pipeline = VoiceActivityDetection(segmentation=model, step=step) pipeline.instantiate({"min_duration_on": 0.0, "min_duration_off": 0.0}) annotation = pipeline(path) else: inference = Inference( model, step=step, pre_aggregation_hook=lambda s: np.partition(s, -2, axis=-1)[:, :, -2, np.newaxis], ) binarize = Binarize( onset=0.5, offset=0.5, min_duration_on=0.0, min_duration_off=0.0 ) annotation = binarize(inference(path)) return [(s.start, s.end) for s in annotation.get_timeline()] def timeline_mask(segments, duration: float, resolution: float = 0.01) -> np.ndarray: n = int(duration / resolution) + 1 mask = np.zeros(n, dtype=bool) for start, end in segments: mask[int(start / resolution) : int(end / resolution) + 1] = True return mask def compare(name: str, got, want, duration: float) -> bool: a, b = timeline_mask(got, duration), timeline_mask(want, duration) n = min(len(a), len(b)) disagree = np.count_nonzero(a[:n] != b[:n]) / n covered = np.count_nonzero(b[:n]) / n print(f"{name}: {len(got)} segments here, {len(want)} from pyannote") print(f" reference covers {covered:6.2%} of the timeline") print(f" disagreement {disagree:6.4%} of the timeline") worst = 0.0 for (s0, e0), (s1, e1) in zip(got, want): worst = max(worst, abs(s0 - s1), abs(e0 - e1)) if len(got) == len(want): print(f" worst boundary shift {worst * 1e3:.1f} ms") ok = disagree <= 1e-3 print(f" {'OK' if ok else 'MISMATCH'}") return ok def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("audio") ap.add_argument("--limit", type=int, default=None, help="seconds of audio to use") ap.add_argument("--step", type=float, default=1.0) args = ap.parse_args() import soundfile as sf info = sf.info(args.audio) if info.samplerate != SAMPLE_RATE: raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz") duration = info.duration if args.limit is None else min(info.duration, args.limit) path = args.audio if args.limit is not None: wav, rate = sf.read(args.audio, dtype="float32", always_2d=True) path = "build/verify_clip.wav" sf.write(path, wav[: args.limit * rate, :1], rate, subtype="PCM_16") ok = True for mode in ("speech", "overlap"): got = run_cli(path, mode, args.step) want = run_pyannote(path, mode, args.step) ok &= compare(mode, got, want, duration) print("VERIFY OK" if ok else "VERIFY FAILED") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())