aboutsummaryrefslogtreecommitdiff
path: root/tools/pipeline.py
blob: c4f675dcdc388fee817b2ebff3dbfcf01acd2a5a (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
"""Sliding-window aggregation, the reference for src/slide.c.

pyannote's Inference cuts a recording into overlapping ten-second windows,
converts each window's powerset output to per-speaker activity, and averages
the windows with a Hamming taper. The taper lets a frame sitting at the edge of
one window, where the LSTM has only seen context on one side, defer to the
window that holds it in the middle.

Run as a script it checks this implementation against pyannote's own Inference
on real audio:

    python tools/pipeline.py <audio.wav>
"""

import sys

import numpy as np

from model import (
    FRAME_STEP,
    NUM_FRAMES,
    NUM_SAMPLES,
    SAMPLE_RATE,
    checkpoint_path,
    powerset_mapping,
)

TRACK_SPEECH, TRACK_OVERLAP, TRACK_COUNT = 0, 1, 2
NUM_TRACKS = 3


def rint_div(n: int, d: int) -> int:
    """Round n/d half to even, matching numpy.rint on the same ratio."""
    q, r = divmod(n, d)
    if 2 * r < d:
        return q
    if 2 * r > d:
        return q + 1
    return q + (q & 1)


def window_starts(num_samples: int, step_samples: int) -> list[int]:
    if num_samples < NUM_SAMPLES:
        return [0]
    whole = (num_samples - NUM_SAMPLES) // step_samples + 1
    starts = [i * step_samples for i in range(whole)]
    if (num_samples - NUM_SAMPLES) % step_samples > 0:
        starts.append(whole * step_samples)
    return starts


def num_aggregated_frames(num_samples: int, step: float) -> int:
    step_samples = int(step * SAMPLE_RATE + 0.5)
    starts = window_starts(num_samples, step_samples)
    frames = rint_div(NUM_SAMPLES + (len(starts) - 1) * step_samples, FRAME_STEP) + 1
    padded = num_samples < NUM_SAMPLES or (num_samples - NUM_SAMPLES) % step_samples > 0
    if padded:
        frames = min(frames, num_samples // FRAME_STEP + 1)
    return frames


def powerset_windows(samples: np.ndarray, step: float, batch: int = 8) -> np.ndarray:
    """[num_windows, NUM_FRAMES, 7] class probabilities from the torch model."""
    import torch
    from pyannote.audio import Model

    torch.set_num_threads(1)
    model = Model.from_pretrained(checkpoint_path())
    model.eval()

    step_samples = int(step * SAMPLE_RATE + 0.5)
    starts = window_starts(len(samples), step_samples)
    windows = np.zeros((len(starts), NUM_SAMPLES), dtype=np.float32)
    for i, s in enumerate(starts):
        chunk = samples[s : s + NUM_SAMPLES]
        windows[i, : len(chunk)] = chunk

    out = []
    with torch.no_grad():
        for i in range(0, len(windows), batch):
            logp = model(torch.from_numpy(windows[i : i + batch])[:, None, :])
            out.append(torch.exp(logp).numpy())
    return np.concatenate(out)


def window_tracks(probs: np.ndarray) -> np.ndarray:
    """[num_windows, NUM_FRAMES, 3] speech / overlap / speaker-count votes.

    The powerset output is decoded by taking the most likely class, which is
    the hard decoding pyannote uses, so each window contributes integers rather
    than probabilities; the averaging across windows is what makes the
    aggregate continuous.
    """
    mapping = powerset_mapping()
    active = mapping[probs.argmax(axis=-1)]
    speakers = active.sum(axis=-1)
    return np.stack(
        [(speakers >= 1).astype(np.float32), (speakers >= 2).astype(np.float32), speakers],
        axis=-1,
    )


def overlap_add(votes: np.ndarray, num_samples: int, step: float) -> np.ndarray:
    """Hamming-weighted average of per-window tracks onto the frame grid."""
    step_samples = int(step * SAMPLE_RATE + 0.5)
    starts = window_starts(num_samples, step_samples)
    frames = num_aggregated_frames(num_samples, step)

    taper = np.hamming(NUM_FRAMES).astype(np.float32)[:, None]
    total = np.zeros((frames, votes.shape[-1]), dtype=np.float32)
    weight = np.zeros((frames, votes.shape[-1]), dtype=np.float32)

    for i, start in enumerate(starts):
        base = rint_div(start, FRAME_STEP)
        n = min(NUM_FRAMES, frames - base)
        if n <= 0:
            continue
        total[base : base + n] += votes[i, :n] * taper[:n]
        weight[base : base + n] += taper[:n]

    return np.where(weight > 1e-12, total / np.maximum(weight, 1e-12), 0.0)


def aggregate_tracks(samples: np.ndarray, step: float = 1.0) -> np.ndarray:
    return overlap_add(
        window_tracks(powerset_windows(samples, step)), len(samples), step
    )


def main() -> int:
    """Checks the speech track against pyannote's own VoiceActivityDetection
    inference, which is the aggregation this file reimplements."""
    import soundfile as sf
    import torch
    from pyannote.audio import Inference, Model

    path = sys.argv[1] if len(sys.argv) > 1 else "testdata_aepyx.wav"
    limit = int(sys.argv[2]) if len(sys.argv) > 2 else 45 * SAMPLE_RATE

    wav, rate = sf.read(path, dtype="float32", always_2d=True)
    samples = np.ascontiguousarray(wav[:limit, 0])
    if rate != SAMPLE_RATE:
        raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz")

    torch.set_num_threads(1)
    model = Model.from_pretrained(checkpoint_path())
    inference = Inference(
        model,
        step=1.0,
        pre_aggregation_hook=lambda s: np.max(s, axis=-1, keepdims=True),
    )
    want = inference(
        {"waveform": torch.from_numpy(samples)[None, :], "sample_rate": SAMPLE_RATE}
    ).data[:, 0]

    got = aggregate_tracks(samples)[:, TRACK_SPEECH]

    print(f"frames: pyannote {len(want)}, here {len(got)}")
    n = min(len(want), len(got))
    diff = np.abs(want[:n] - got[:n]).max()
    print(f"max |speech score diff| over {n} frames: {diff:.3e}")

    ok = len(want) == len(got) and diff <= 1e-5
    print("AGGREGATION OK" if ok else "AGGREGATION MISMATCH")
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())