1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
"""Sliding-window aggregation, the reference for src/slide.c.
pyannote's Inference cuts a recording into overlapping ten-second windows,
converts each window's powerset output to per-speaker activity, and averages
the windows with a Hamming taper. The taper lets a frame sitting at the edge of
one window, where the LSTM has only seen context on one side, defer to the
window that holds it in the middle.
Run as a script it checks this implementation against pyannote's own Inference
on real audio:
python tools/pipeline.py <audio.wav>
"""
import sys
import numpy as np
from model import (
FRAME_STEP,
NUM_FRAMES,
NUM_SAMPLES,
SAMPLE_RATE,
checkpoint_path,
powerset_mapping,
)
TRACK_SPEECH, TRACK_OVERLAP, TRACK_COUNT = 0, 1, 2
NUM_TRACKS = 3
def rint_div(n: int, d: int) -> int:
"""Round n/d half to even, matching numpy.rint on the same ratio."""
q, r = divmod(n, d)
if 2 * r < d:
return q
if 2 * r > d:
return q + 1
return q + (q & 1)
def window_starts(num_samples: int, step_samples: int) -> list[int]:
if num_samples < NUM_SAMPLES:
return [0]
whole = (num_samples - NUM_SAMPLES) // step_samples + 1
starts = [i * step_samples for i in range(whole)]
if (num_samples - NUM_SAMPLES) % step_samples > 0:
starts.append(whole * step_samples)
return starts
def num_aggregated_frames(num_samples: int, step: float) -> int:
step_samples = int(step * SAMPLE_RATE + 0.5)
starts = window_starts(num_samples, step_samples)
frames = rint_div(NUM_SAMPLES + (len(starts) - 1) * step_samples, FRAME_STEP) + 1
padded = num_samples < NUM_SAMPLES or (num_samples - NUM_SAMPLES) % step_samples > 0
if padded:
frames = min(frames, num_samples // FRAME_STEP + 1)
return frames
def powerset_windows(samples: np.ndarray, step: float, batch: int = 8) -> np.ndarray:
"""[num_windows, NUM_FRAMES, 7] class probabilities from the torch model."""
import torch
from pyannote.audio import Model
torch.set_num_threads(1)
model = Model.from_pretrained(checkpoint_path())
model.eval()
step_samples = int(step * SAMPLE_RATE + 0.5)
starts = window_starts(len(samples), step_samples)
windows = np.zeros((len(starts), NUM_SAMPLES), dtype=np.float32)
for i, s in enumerate(starts):
chunk = samples[s : s + NUM_SAMPLES]
windows[i, : len(chunk)] = chunk
out = []
with torch.no_grad():
for i in range(0, len(windows), batch):
logp = model(torch.from_numpy(windows[i : i + batch])[:, None, :])
out.append(torch.exp(logp).numpy())
return np.concatenate(out)
def window_tracks(probs: np.ndarray) -> np.ndarray:
"""[num_windows, NUM_FRAMES, 3] speech / overlap / speaker-count votes.
The powerset output is decoded by taking the most likely class, which is
the hard decoding pyannote uses, so each window contributes integers rather
than probabilities; the averaging across windows is what makes the
aggregate continuous.
"""
mapping = powerset_mapping()
active = mapping[probs.argmax(axis=-1)]
speakers = active.sum(axis=-1)
return np.stack(
[(speakers >= 1).astype(np.float32), (speakers >= 2).astype(np.float32), speakers],
axis=-1,
)
def overlap_add(votes: np.ndarray, num_samples: int, step: float) -> np.ndarray:
"""Hamming-weighted average of per-window tracks onto the frame grid."""
step_samples = int(step * SAMPLE_RATE + 0.5)
starts = window_starts(num_samples, step_samples)
frames = num_aggregated_frames(num_samples, step)
taper = np.hamming(NUM_FRAMES).astype(np.float32)[:, None]
total = np.zeros((frames, votes.shape[-1]), dtype=np.float32)
weight = np.zeros((frames, votes.shape[-1]), dtype=np.float32)
for i, start in enumerate(starts):
base = rint_div(start, FRAME_STEP)
n = min(NUM_FRAMES, frames - base)
if n <= 0:
continue
total[base : base + n] += votes[i, :n] * taper[:n]
weight[base : base + n] += taper[:n]
return np.where(weight > 1e-12, total / np.maximum(weight, 1e-12), 0.0)
def aggregate_tracks(samples: np.ndarray, step: float = 1.0) -> np.ndarray:
return overlap_add(
window_tracks(powerset_windows(samples, step)), len(samples), step
)
def main() -> int:
"""Checks the speech track against pyannote's own VoiceActivityDetection
inference, which is the aggregation this file reimplements."""
import soundfile as sf
import torch
from pyannote.audio import Inference, Model
path = sys.argv[1] if len(sys.argv) > 1 else "testdata_aepyx.wav"
limit = int(sys.argv[2]) if len(sys.argv) > 2 else 45 * SAMPLE_RATE
wav, rate = sf.read(path, dtype="float32", always_2d=True)
samples = np.ascontiguousarray(wav[:limit, 0])
if rate != SAMPLE_RATE:
raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz")
torch.set_num_threads(1)
model = Model.from_pretrained(checkpoint_path())
inference = Inference(
model,
step=1.0,
pre_aggregation_hook=lambda s: np.max(s, axis=-1, keepdims=True),
)
want = inference(
{"waveform": torch.from_numpy(samples)[None, :], "sample_rate": SAMPLE_RATE}
).data[:, 0]
got = aggregate_tracks(samples)[:, TRACK_SPEECH]
print(f"frames: pyannote {len(want)}, here {len(got)}")
n = min(len(want), len(got))
diff = np.abs(want[:n] - got[:n]).max()
print(f"max |speech score diff| over {n} frames: {diff:.3e}")
ok = len(want) == len(got) and diff <= 1e-5
print("AGGREGATION OK" if ok else "AGGREGATION MISMATCH")
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())
|