aboutsummaryrefslogtreecommitdiff
path: root/tools/verify_cli.py
blob: 15e7cefa83a0d47a82d6dc867c768e388a08495c (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
"""End-to-end check of the built binary against pyannote's own pipelines.

The parity tests compare tensors; this compares what a user actually gets. It
derives speech and overlapped-speech segments from pyannote for a recording,
runs pyseg_cli over the same file, and reports both the segment boundaries
and the fraction of the timeline the two disagree about, which is the number
that matters for anything downstream.

    python tools/verify_cli.py <audio.wav> [--limit SECONDS] [--step SECONDS]
"""

import argparse
import subprocess
import sys

import numpy as np

from model import SAMPLE_RATE, checkpoint_path

BINARY = "./pyseg_cli"


def run_cli(path: str, mode: str, step: float) -> list[tuple[float, float]]:
    out = subprocess.run(
        [BINARY, "--mode", mode, "--step", str(step), path],
        check=True,
        capture_output=True,
        text=True,
    ).stdout
    return [
        (float(a), float(b))
        for a, b in (line.split() for line in out.splitlines() if line.strip())
    ]


def run_pyannote(path: str, mode: str, step: float):
    """Reference segments for one track.

    Speech goes through pyannote's own VoiceActivityDetection pipeline. Overlap
    is assembled here from the same parts, because the pipeline that used to
    wrap them was dropped from pyannote.audio: take the second largest
    per-speaker score of each frame, which for this model's hard multi-label
    output is one exactly when at least two speakers are active, aggregate, and
    binarize with the same thresholds.
    """
    import torch
    from pyannote.audio import Inference, Model
    from pyannote.audio.pipelines import VoiceActivityDetection
    from pyannote.audio.utils.signal import Binarize

    torch.set_num_threads(1)
    model = Model.from_pretrained(checkpoint_path())

    if mode == "speech":
        pipeline = VoiceActivityDetection(segmentation=model, step=step)
        pipeline.instantiate({"min_duration_on": 0.0, "min_duration_off": 0.0})
        annotation = pipeline(path)
    else:
        inference = Inference(
            model,
            step=step,
            pre_aggregation_hook=lambda s: np.partition(s, -2, axis=-1)[:, :, -2, np.newaxis],
        )
        binarize = Binarize(
            onset=0.5, offset=0.5, min_duration_on=0.0, min_duration_off=0.0
        )
        annotation = binarize(inference(path))

    return [(s.start, s.end) for s in annotation.get_timeline()]


def timeline_mask(segments, duration: float, resolution: float = 0.01) -> np.ndarray:
    n = int(duration / resolution) + 1
    mask = np.zeros(n, dtype=bool)
    for start, end in segments:
        mask[int(start / resolution) : int(end / resolution) + 1] = True
    return mask


def compare(name: str, got, want, duration: float) -> bool:
    a, b = timeline_mask(got, duration), timeline_mask(want, duration)
    n = min(len(a), len(b))
    disagree = np.count_nonzero(a[:n] != b[:n]) / n
    covered = np.count_nonzero(b[:n]) / n

    print(f"{name}: {len(got)} segments here, {len(want)} from pyannote")
    print(f"  reference covers {covered:6.2%} of the timeline")
    print(f"  disagreement     {disagree:6.4%} of the timeline")

    worst = 0.0
    for (s0, e0), (s1, e1) in zip(got, want):
        worst = max(worst, abs(s0 - s1), abs(e0 - e1))
    if len(got) == len(want):
        print(f"  worst boundary shift {worst * 1e3:.1f} ms")

    ok = disagree <= 1e-3
    print(f"  {'OK' if ok else 'MISMATCH'}")
    return ok


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("audio")
    ap.add_argument("--limit", type=int, default=None, help="seconds of audio to use")
    ap.add_argument("--step", type=float, default=1.0)
    args = ap.parse_args()

    import soundfile as sf

    info = sf.info(args.audio)
    if info.samplerate != SAMPLE_RATE:
        raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz")
    duration = info.duration if args.limit is None else min(info.duration, args.limit)

    path = args.audio
    if args.limit is not None:
        wav, rate = sf.read(args.audio, dtype="float32", always_2d=True)
        path = "build/verify_clip.wav"
        sf.write(path, wav[: args.limit * rate, :1], rate, subtype="PCM_16")

    ok = True
    for mode in ("speech", "overlap"):
        got = run_cli(path, mode, args.step)
        want = run_pyannote(path, mode, args.step)
        ok &= compare(mode, got, want, duration)

    print("VERIFY OK" if ok else "VERIFY FAILED")
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())