1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
"""Loading and structural description of pyannote/segmentation-3.0.
The checkpoint is a PyTorch Lightning file whose pickle references pyannote
classes, so reading it requires pyannote.audio to be importable. Everything
downstream of this module works on plain numpy arrays.
"""
from pathlib import Path
import numpy as np
REPO = "pyannote/segmentation-3.0"
SNAPSHOT = (
Path.home()
/ ".cache/huggingface/hub/models--pyannote--segmentation-3.0/snapshots"
)
SAMPLE_RATE = 16000
DURATION = 10.0
NUM_SAMPLES = 160000
NUM_FRAMES = 589
NUM_CLASSES = 7
HIDDEN = 128
NUM_LAYERS = 4
# SincNet geometry: (kernel_size, stride) for conv, abs, pool, conv, pool, conv, pool.
SINC_KERNEL = 251
SINC_STRIDE = 10
SINC_FILTERS = 80
def checkpoint_path() -> Path:
"""Path of the single snapshot of the model in the local Hugging Face cache."""
snaps = sorted(SNAPSHOT.glob("*/pytorch_model.bin"))
if not snaps:
raise FileNotFoundError(
f"{REPO} not found under {SNAPSHOT}; run \"hf download {REPO}\" first"
)
return snaps[-1]
def load_state_dict() -> dict:
"""Returns the checkpoint tensors as numpy arrays keyed by parameter name."""
import torch
ckpt = torch.load(checkpoint_path(), map_location="cpu", weights_only=False)
return {k: v.numpy().copy() for k, v in ckpt["state_dict"].items()}
def sinc_filters(sd: dict) -> np.ndarray:
"""Materialises the parametric sinc filterbank as a plain [80, 251] matrix.
asteroid_filterbanks builds the filters from the learned band edges every
forward pass, but the edges are frozen at inference time, so the result is a
constant that can be folded into the exported weights.
"""
low_hz = sd["sincnet.conv1d.0.filterbank.low_hz_"].astype(np.float64)
band_hz = sd["sincnet.conv1d.0.filterbank.band_hz_"].astype(np.float64)
window = sd["sincnet.conv1d.0.filterbank.window_"].astype(np.float64)
n_ = sd["sincnet.conv1d.0.filterbank.n_"].astype(np.float64)
min_low_hz, min_band_hz = 50.0, 50.0
low = min_low_hz + np.abs(low_hz)
high = np.clip(low + min_band_hz + np.abs(band_hz), min_low_hz, SAMPLE_RATE / 2)
band = (high - low)[:, 0]
ft_low = low @ n_
ft_high = high @ n_
cos_left = ((np.sin(ft_high) - np.sin(ft_low)) / (n_ / 2)) * window
cos = np.concatenate(
[cos_left, 2 * band[:, None], np.flip(cos_left, axis=1)], axis=1
)
cos /= 2 * band[:, None]
sin_left = ((np.cos(ft_low) - np.cos(ft_high)) / (n_ / 2)) * window
sin = np.concatenate(
[sin_left, np.zeros_like(band[:, None]), -np.flip(sin_left, axis=1)], axis=1
)
sin /= 2 * band[:, None]
return np.concatenate([cos, sin], axis=0).astype(np.float32)
def num_frames(num_samples: int) -> int:
"""Output frame count of the SincNet stack for a given input length."""
geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)]
n = num_samples
for kernel, stride in geometry:
n = 1 + (n - kernel) // stride
return n
def receptive_field() -> tuple[int, int, int]:
"""(size, step, start) of the output frame grid, in samples.
Frame f covers the 991 input samples starting at f * 270. The grid begins
exactly at sample 0 because the centre of the first frame sits half a
receptive field into the signal.
"""
geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)]
def rf_size(frames: int) -> int:
size = frames
for kernel, stride in reversed(geometry):
size = kernel + (size - 1) * stride
return size
def rf_centre(frame: int) -> int:
centre = frame
for kernel, stride in reversed(geometry):
centre = centre * stride + (kernel - 1) // 2
return centre
size = rf_size(1)
step = rf_size(2) - size
start = rf_centre(0) - (size - 1) // 2
return size, step, start
def powerset_mapping(num_classes: int = 3, max_set_size: int = 2) -> np.ndarray:
"""[num_powerset_classes, num_classes] membership matrix in pyannote order."""
from itertools import combinations
rows = []
for size in range(max_set_size + 1):
for members in combinations(range(num_classes), size):
row = np.zeros(num_classes, dtype=np.float32)
row[list(members)] = 1.0
rows.append(row)
return np.stack(rows)
FRAME_SIZE, FRAME_STEP, FRAME_START = receptive_field()
assert num_frames(NUM_SAMPLES) == NUM_FRAMES
assert (FRAME_SIZE, FRAME_STEP, FRAME_START) == (991, 270, 0)
|