aboutsummaryrefslogtreecommitdiff
path: root/tools/model.py
blob: b6601e34a93546ac2c5622b933914d8dfc9b20c4 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
"""Loading and structural description of pyannote/segmentation-3.0.

The checkpoint is a PyTorch Lightning file whose pickle references pyannote
classes, so reading it requires pyannote.audio to be importable. Everything
downstream of this module works on plain numpy arrays.
"""

from pathlib import Path

import numpy as np

REPO = "pyannote/segmentation-3.0"
SNAPSHOT = (
    Path.home()
    / ".cache/huggingface/hub/models--pyannote--segmentation-3.0/snapshots"
)

SAMPLE_RATE = 16000
DURATION = 10.0
NUM_SAMPLES = 160000
NUM_FRAMES = 589
NUM_CLASSES = 7
HIDDEN = 128
NUM_LAYERS = 4

# SincNet geometry: (kernel_size, stride) for conv, abs, pool, conv, pool, conv, pool.
SINC_KERNEL = 251
SINC_STRIDE = 10
SINC_FILTERS = 80


def checkpoint_path() -> Path:
    """Path of the single snapshot of the model in the local Hugging Face cache."""
    snaps = sorted(SNAPSHOT.glob("*/pytorch_model.bin"))
    if not snaps:
        raise FileNotFoundError(
            f"{REPO} not found under {SNAPSHOT}; run \"hf download {REPO}\" first"
        )
    return snaps[-1]


def load_state_dict() -> dict:
    """Returns the checkpoint tensors as numpy arrays keyed by parameter name."""
    import torch

    ckpt = torch.load(checkpoint_path(), map_location="cpu", weights_only=False)
    return {k: v.numpy().copy() for k, v in ckpt["state_dict"].items()}


def sinc_filters(sd: dict) -> np.ndarray:
    """Materialises the parametric sinc filterbank as a plain [80, 251] matrix.

    asteroid_filterbanks builds the filters from the learned band edges every
    forward pass, but the edges are frozen at inference time, so the result is a
    constant that can be folded into the exported weights.
    """
    low_hz = sd["sincnet.conv1d.0.filterbank.low_hz_"].astype(np.float64)
    band_hz = sd["sincnet.conv1d.0.filterbank.band_hz_"].astype(np.float64)
    window = sd["sincnet.conv1d.0.filterbank.window_"].astype(np.float64)
    n_ = sd["sincnet.conv1d.0.filterbank.n_"].astype(np.float64)

    min_low_hz, min_band_hz = 50.0, 50.0
    low = min_low_hz + np.abs(low_hz)
    high = np.clip(low + min_band_hz + np.abs(band_hz), min_low_hz, SAMPLE_RATE / 2)
    band = (high - low)[:, 0]

    ft_low = low @ n_
    ft_high = high @ n_

    cos_left = ((np.sin(ft_high) - np.sin(ft_low)) / (n_ / 2)) * window
    cos = np.concatenate(
        [cos_left, 2 * band[:, None], np.flip(cos_left, axis=1)], axis=1
    )
    cos /= 2 * band[:, None]

    sin_left = ((np.cos(ft_low) - np.cos(ft_high)) / (n_ / 2)) * window
    sin = np.concatenate(
        [sin_left, np.zeros_like(band[:, None]), -np.flip(sin_left, axis=1)], axis=1
    )
    sin /= 2 * band[:, None]

    return np.concatenate([cos, sin], axis=0).astype(np.float32)


def num_frames(num_samples: int) -> int:
    """Output frame count of the SincNet stack for a given input length."""
    geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)]
    n = num_samples
    for kernel, stride in geometry:
        n = 1 + (n - kernel) // stride
    return n


def receptive_field() -> tuple[int, int, int]:
    """(size, step, start) of the output frame grid, in samples.

    Frame f covers the 991 input samples starting at f * 270. The grid begins
    exactly at sample 0 because the centre of the first frame sits half a
    receptive field into the signal.
    """
    geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)]

    def rf_size(frames: int) -> int:
        size = frames
        for kernel, stride in reversed(geometry):
            size = kernel + (size - 1) * stride
        return size

    def rf_centre(frame: int) -> int:
        centre = frame
        for kernel, stride in reversed(geometry):
            centre = centre * stride + (kernel - 1) // 2
        return centre

    size = rf_size(1)
    step = rf_size(2) - size
    start = rf_centre(0) - (size - 1) // 2
    return size, step, start


def powerset_mapping(num_classes: int = 3, max_set_size: int = 2) -> np.ndarray:
    """[num_powerset_classes, num_classes] membership matrix in pyannote order."""
    from itertools import combinations

    rows = []
    for size in range(max_set_size + 1):
        for members in combinations(range(num_classes), size):
            row = np.zeros(num_classes, dtype=np.float32)
            row[list(members)] = 1.0
            rows.append(row)
    return np.stack(rows)


FRAME_SIZE, FRAME_STEP, FRAME_START = receptive_field()

assert num_frames(NUM_SAMPLES) == NUM_FRAMES
assert (FRAME_SIZE, FRAME_STEP, FRAME_START) == (991, 270, 0)