diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:03:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:13:28 +0200 |
| commit | 64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch) | |
| tree | 85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /tools/model.py | |
| download | pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip | |
Init
Diffstat (limited to 'tools/model.py')
| -rw-r--r-- | tools/model.py | 137 |
1 files changed, 137 insertions, 0 deletions
diff --git a/tools/model.py b/tools/model.py new file mode 100644 index 0000000..b6601e3 --- /dev/null +++ b/tools/model.py @@ -0,0 +1,137 @@ +"""Loading and structural description of pyannote/segmentation-3.0. + +The checkpoint is a PyTorch Lightning file whose pickle references pyannote +classes, so reading it requires pyannote.audio to be importable. Everything +downstream of this module works on plain numpy arrays. +""" + +from pathlib import Path + +import numpy as np + +REPO = "pyannote/segmentation-3.0" +SNAPSHOT = ( + Path.home() + / ".cache/huggingface/hub/models--pyannote--segmentation-3.0/snapshots" +) + +SAMPLE_RATE = 16000 +DURATION = 10.0 +NUM_SAMPLES = 160000 +NUM_FRAMES = 589 +NUM_CLASSES = 7 +HIDDEN = 128 +NUM_LAYERS = 4 + +# SincNet geometry: (kernel_size, stride) for conv, abs, pool, conv, pool, conv, pool. +SINC_KERNEL = 251 +SINC_STRIDE = 10 +SINC_FILTERS = 80 + + +def checkpoint_path() -> Path: + """Path of the single snapshot of the model in the local Hugging Face cache.""" + snaps = sorted(SNAPSHOT.glob("*/pytorch_model.bin")) + if not snaps: + raise FileNotFoundError( + f"{REPO} not found under {SNAPSHOT}; run \"hf download {REPO}\" first" + ) + return snaps[-1] + + +def load_state_dict() -> dict: + """Returns the checkpoint tensors as numpy arrays keyed by parameter name.""" + import torch + + ckpt = torch.load(checkpoint_path(), map_location="cpu", weights_only=False) + return {k: v.numpy().copy() for k, v in ckpt["state_dict"].items()} + + +def sinc_filters(sd: dict) -> np.ndarray: + """Materialises the parametric sinc filterbank as a plain [80, 251] matrix. + + asteroid_filterbanks builds the filters from the learned band edges every + forward pass, but the edges are frozen at inference time, so the result is a + constant that can be folded into the exported weights. + """ + low_hz = sd["sincnet.conv1d.0.filterbank.low_hz_"].astype(np.float64) + band_hz = sd["sincnet.conv1d.0.filterbank.band_hz_"].astype(np.float64) + window = sd["sincnet.conv1d.0.filterbank.window_"].astype(np.float64) + n_ = sd["sincnet.conv1d.0.filterbank.n_"].astype(np.float64) + + min_low_hz, min_band_hz = 50.0, 50.0 + low = min_low_hz + np.abs(low_hz) + high = np.clip(low + min_band_hz + np.abs(band_hz), min_low_hz, SAMPLE_RATE / 2) + band = (high - low)[:, 0] + + ft_low = low @ n_ + ft_high = high @ n_ + + cos_left = ((np.sin(ft_high) - np.sin(ft_low)) / (n_ / 2)) * window + cos = np.concatenate( + [cos_left, 2 * band[:, None], np.flip(cos_left, axis=1)], axis=1 + ) + cos /= 2 * band[:, None] + + sin_left = ((np.cos(ft_low) - np.cos(ft_high)) / (n_ / 2)) * window + sin = np.concatenate( + [sin_left, np.zeros_like(band[:, None]), -np.flip(sin_left, axis=1)], axis=1 + ) + sin /= 2 * band[:, None] + + return np.concatenate([cos, sin], axis=0).astype(np.float32) + + +def num_frames(num_samples: int) -> int: + """Output frame count of the SincNet stack for a given input length.""" + geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)] + n = num_samples + for kernel, stride in geometry: + n = 1 + (n - kernel) // stride + return n + + +def receptive_field() -> tuple[int, int, int]: + """(size, step, start) of the output frame grid, in samples. + + Frame f covers the 991 input samples starting at f * 270. The grid begins + exactly at sample 0 because the centre of the first frame sits half a + receptive field into the signal. + """ + geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)] + + def rf_size(frames: int) -> int: + size = frames + for kernel, stride in reversed(geometry): + size = kernel + (size - 1) * stride + return size + + def rf_centre(frame: int) -> int: + centre = frame + for kernel, stride in reversed(geometry): + centre = centre * stride + (kernel - 1) // 2 + return centre + + size = rf_size(1) + step = rf_size(2) - size + start = rf_centre(0) - (size - 1) // 2 + return size, step, start + + +def powerset_mapping(num_classes: int = 3, max_set_size: int = 2) -> np.ndarray: + """[num_powerset_classes, num_classes] membership matrix in pyannote order.""" + from itertools import combinations + + rows = [] + for size in range(max_set_size + 1): + for members in combinations(range(num_classes), size): + row = np.zeros(num_classes, dtype=np.float32) + row[list(members)] = 1.0 + rows.append(row) + return np.stack(rows) + + +FRAME_SIZE, FRAME_STEP, FRAME_START = receptive_field() + +assert num_frames(NUM_SAMPLES) == NUM_FRAMES +assert (FRAME_SIZE, FRAME_STEP, FRAME_START) == (991, 270, 0) |
