"""Numpy forward pass for pyannote/segmentation-3.0. This is the executable specification the C engine is written against: every operator appears here in the order and with the exact arithmetic the checkpoint was trained with. Run it against the torch model to confirm the two agree: .venv/bin/python tools/reference.py The stage outputs are returned as well, so a failing parity run can be bisected to a single stage instead of guessed at. """ import sys import numpy as np from model import ( HIDDEN, NUM_FRAMES, NUM_SAMPLES, SINC_FILTERS, load_state_dict, sinc_filters, ) EPS = 1e-5 LEAKY = 0.01 def instance_norm(x: np.ndarray, weight: np.ndarray, bias: np.ndarray) -> np.ndarray: """InstanceNorm1d(affine=True) over [channels, time]; variance is biased.""" mean = x.mean(axis=-1, keepdims=True) var = x.var(axis=-1, keepdims=True) return (x - mean) / np.sqrt(var + EPS) * weight[:, None] + bias[:, None] def leaky_relu(x: np.ndarray) -> np.ndarray: return np.where(x >= 0, x, LEAKY * x) def max_pool(x: np.ndarray, size: int = 3, stride: int = 3) -> np.ndarray: """MaxPool1d over the time axis of a [channels, time] array.""" n = 1 + (x.shape[-1] - size) // stride view = np.lib.stride_tricks.as_strided( x, shape=(x.shape[0], n, size), strides=(x.strides[0], x.strides[1] * stride, x.strides[1]), ) return view.max(axis=-1) def conv1d(x: np.ndarray, weight: np.ndarray, bias: np.ndarray, stride: int = 1): """Valid 1D convolution of [in_ch, time] with [out_ch, in_ch, kernel].""" in_ch, length = x.shape out_ch, _, kernel = weight.shape n = 1 + (length - kernel) // stride # [in_ch, kernel, n] gather, contracted against the weight in one matmul. cols = np.lib.stride_tricks.as_strided( x, shape=(in_ch, kernel, n), strides=(x.strides[0], x.strides[1], x.strides[1] * stride), ).reshape(in_ch * kernel, n) return weight.reshape(out_ch, in_ch * kernel) @ cols + bias[:, None] def lstm_layer(x, w_ih, w_hh, bias, reverse: bool): """One direction of a PyTorch LSTM layer over [time, features].""" steps = x.shape[0] h = np.zeros(HIDDEN, dtype=np.float32) c = np.zeros(HIDDEN, dtype=np.float32) pre = x @ w_ih.T + bias out = np.empty((steps, HIDDEN), dtype=np.float32) order = range(steps - 1, -1, -1) if reverse else range(steps) for t in order: g = pre[t] + h @ w_hh.T i = sigmoid(g[0:HIDDEN]) f = sigmoid(g[HIDDEN : 2 * HIDDEN]) gg = np.tanh(g[2 * HIDDEN : 3 * HIDDEN]) o = sigmoid(g[3 * HIDDEN : 4 * HIDDEN]) c = f * c + i * gg h = o * np.tanh(c) out[t] = h return out def sigmoid(x): return 1.0 / (1.0 + np.exp(-x)) def log_softmax(x): m = x.max(axis=-1, keepdims=True) z = x - m return z - np.log(np.exp(z).sum(axis=-1, keepdims=True)) class Reference: def __init__(self, sd: dict): self.sd = sd self.sinc = sinc_filters(sd)[:, None, :] def sincnet(self, wav: np.ndarray, stages: dict) -> np.ndarray: sd = self.sd x = wav[None, :] x = instance_norm(x, sd["sincnet.wav_norm1d.weight"], sd["sincnet.wav_norm1d.bias"]) stages["wav_norm"] = x x = conv1d(x, self.sinc, np.zeros(SINC_FILTERS, dtype=np.float32), stride=10) x = np.abs(x) stages["sinc"] = x x = max_pool(x) stages["pool1"] = x x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.0.weight"], sd["sincnet.norm1d.0.bias"])) stages["block1"] = x x = conv1d(x, sd["sincnet.conv1d.1.weight"], sd["sincnet.conv1d.1.bias"]) stages["conv2"] = x x = max_pool(x) x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.1.weight"], sd["sincnet.norm1d.1.bias"])) stages["block2"] = x x = conv1d(x, sd["sincnet.conv1d.2.weight"], sd["sincnet.conv1d.2.bias"]) x = max_pool(x) x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.2.weight"], sd["sincnet.norm1d.2.bias"])) stages["block3"] = x return x def forward(self, wav: np.ndarray, stages: dict | None = None) -> np.ndarray: """[num_samples] float32 waveform -> [num_frames, 7] log probabilities.""" stages = {} if stages is None else stages sd = self.sd x = self.sincnet(wav, stages).T # [time, 60] for layer in range(4): suffix = f"l{layer}" fwd = lstm_layer( x, sd[f"lstm.weight_ih_{suffix}"], sd[f"lstm.weight_hh_{suffix}"], sd[f"lstm.bias_ih_{suffix}"] + sd[f"lstm.bias_hh_{suffix}"], reverse=False, ) bwd = lstm_layer( x, sd[f"lstm.weight_ih_{suffix}_reverse"], sd[f"lstm.weight_hh_{suffix}_reverse"], sd[f"lstm.bias_ih_{suffix}_reverse"] + sd[f"lstm.bias_hh_{suffix}_reverse"], reverse=True, ) x = np.concatenate([fwd, bwd], axis=-1) stages[f"lstm{layer}"] = x for i in range(2): x = leaky_relu(x @ sd[f"linear.{i}.weight"].T + sd[f"linear.{i}.bias"]) stages[f"linear{i}"] = x logits = x @ sd["classifier.weight"].T + sd["classifier.bias"] stages["logits"] = logits return log_softmax(logits) def main() -> int: import torch from pyannote.audio import Model from model import checkpoint_path sd = load_state_dict() ref = Reference(sd) torch_model = Model.from_pretrained(checkpoint_path()) torch_model.eval() rng = np.random.default_rng(0) worst = 0.0 for trial in range(3): wav = rng.standard_normal(NUM_SAMPLES).astype(np.float32) * 0.1 with torch.no_grad(): want = torch_model(torch.from_numpy(wav)[None, None, :]).numpy()[0] got = ref.forward(wav) assert got.shape == (NUM_FRAMES, 7), got.shape diff = np.abs(got - want).max() worst = max(worst, diff) print(f"trial {trial}: max |log-prob diff| = {diff:.3e}") ok = worst <= 2e-4 print(f"worst {worst:.3e} -> {'PARITY OK' if ok else 'PARITY FAILED'}") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())