aboutsummaryrefslogtreecommitdiff
path: root/tools/verify_cli.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/verify_cli.py')
-rw-r--r--tools/verify_cli.py132
1 files changed, 132 insertions, 0 deletions
diff --git a/tools/verify_cli.py b/tools/verify_cli.py
new file mode 100644
index 0000000..15e7cef
--- /dev/null
+++ b/tools/verify_cli.py
@@ -0,0 +1,132 @@
+"""End-to-end check of the built binary against pyannote's own pipelines.
+
+The parity tests compare tensors; this compares what a user actually gets. It
+derives speech and overlapped-speech segments from pyannote for a recording,
+runs pyseg_cli over the same file, and reports both the segment boundaries
+and the fraction of the timeline the two disagree about, which is the number
+that matters for anything downstream.
+
+ python tools/verify_cli.py <audio.wav> [--limit SECONDS] [--step SECONDS]
+"""
+
+import argparse
+import subprocess
+import sys
+
+import numpy as np
+
+from model import SAMPLE_RATE, checkpoint_path
+
+BINARY = "./pyseg_cli"
+
+
+def run_cli(path: str, mode: str, step: float) -> list[tuple[float, float]]:
+ out = subprocess.run(
+ [BINARY, "--mode", mode, "--step", str(step), path],
+ check=True,
+ capture_output=True,
+ text=True,
+ ).stdout
+ return [
+ (float(a), float(b))
+ for a, b in (line.split() for line in out.splitlines() if line.strip())
+ ]
+
+
+def run_pyannote(path: str, mode: str, step: float):
+ """Reference segments for one track.
+
+ Speech goes through pyannote's own VoiceActivityDetection pipeline. Overlap
+ is assembled here from the same parts, because the pipeline that used to
+ wrap them was dropped from pyannote.audio: take the second largest
+ per-speaker score of each frame, which for this model's hard multi-label
+ output is one exactly when at least two speakers are active, aggregate, and
+ binarize with the same thresholds.
+ """
+ import torch
+ from pyannote.audio import Inference, Model
+ from pyannote.audio.pipelines import VoiceActivityDetection
+ from pyannote.audio.utils.signal import Binarize
+
+ torch.set_num_threads(1)
+ model = Model.from_pretrained(checkpoint_path())
+
+ if mode == "speech":
+ pipeline = VoiceActivityDetection(segmentation=model, step=step)
+ pipeline.instantiate({"min_duration_on": 0.0, "min_duration_off": 0.0})
+ annotation = pipeline(path)
+ else:
+ inference = Inference(
+ model,
+ step=step,
+ pre_aggregation_hook=lambda s: np.partition(s, -2, axis=-1)[:, :, -2, np.newaxis],
+ )
+ binarize = Binarize(
+ onset=0.5, offset=0.5, min_duration_on=0.0, min_duration_off=0.0
+ )
+ annotation = binarize(inference(path))
+
+ return [(s.start, s.end) for s in annotation.get_timeline()]
+
+
+def timeline_mask(segments, duration: float, resolution: float = 0.01) -> np.ndarray:
+ n = int(duration / resolution) + 1
+ mask = np.zeros(n, dtype=bool)
+ for start, end in segments:
+ mask[int(start / resolution) : int(end / resolution) + 1] = True
+ return mask
+
+
+def compare(name: str, got, want, duration: float) -> bool:
+ a, b = timeline_mask(got, duration), timeline_mask(want, duration)
+ n = min(len(a), len(b))
+ disagree = np.count_nonzero(a[:n] != b[:n]) / n
+ covered = np.count_nonzero(b[:n]) / n
+
+ print(f"{name}: {len(got)} segments here, {len(want)} from pyannote")
+ print(f" reference covers {covered:6.2%} of the timeline")
+ print(f" disagreement {disagree:6.4%} of the timeline")
+
+ worst = 0.0
+ for (s0, e0), (s1, e1) in zip(got, want):
+ worst = max(worst, abs(s0 - s1), abs(e0 - e1))
+ if len(got) == len(want):
+ print(f" worst boundary shift {worst * 1e3:.1f} ms")
+
+ ok = disagree <= 1e-3
+ print(f" {'OK' if ok else 'MISMATCH'}")
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("audio")
+ ap.add_argument("--limit", type=int, default=None, help="seconds of audio to use")
+ ap.add_argument("--step", type=float, default=1.0)
+ args = ap.parse_args()
+
+ import soundfile as sf
+
+ info = sf.info(args.audio)
+ if info.samplerate != SAMPLE_RATE:
+ raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz")
+ duration = info.duration if args.limit is None else min(info.duration, args.limit)
+
+ path = args.audio
+ if args.limit is not None:
+ wav, rate = sf.read(args.audio, dtype="float32", always_2d=True)
+ path = "build/verify_clip.wav"
+ sf.write(path, wav[: args.limit * rate, :1], rate, subtype="PCM_16")
+
+ ok = True
+ for mode in ("speech", "overlap"):
+ got = run_cli(path, mode, args.step)
+ want = run_pyannote(path, mode, args.step)
+ ok &= compare(mode, got, want, duration)
+
+ print("VERIFY OK" if ok else "VERIFY FAILED")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())