diff options
Diffstat (limited to 'tools/reference.py')
| -rw-r--r-- | tools/reference.py | 192 |
1 files changed, 192 insertions, 0 deletions
diff --git a/tools/reference.py b/tools/reference.py new file mode 100644 index 0000000..1992f7f --- /dev/null +++ b/tools/reference.py @@ -0,0 +1,192 @@ +"""Numpy forward pass for pyannote/segmentation-3.0. + +This is the executable specification the C engine is written against: every +operator appears here in the order and with the exact arithmetic the checkpoint +was trained with. Run it against the torch model to confirm the two agree: + + .venv/bin/python tools/reference.py + +The stage outputs are returned as well, so a failing parity run can be bisected +to a single stage instead of guessed at. +""" + +import sys + +import numpy as np + +from model import ( + HIDDEN, + NUM_FRAMES, + NUM_SAMPLES, + SINC_FILTERS, + load_state_dict, + sinc_filters, +) + +EPS = 1e-5 +LEAKY = 0.01 + + +def instance_norm(x: np.ndarray, weight: np.ndarray, bias: np.ndarray) -> np.ndarray: + """InstanceNorm1d(affine=True) over [channels, time]; variance is biased.""" + mean = x.mean(axis=-1, keepdims=True) + var = x.var(axis=-1, keepdims=True) + return (x - mean) / np.sqrt(var + EPS) * weight[:, None] + bias[:, None] + + +def leaky_relu(x: np.ndarray) -> np.ndarray: + return np.where(x >= 0, x, LEAKY * x) + + +def max_pool(x: np.ndarray, size: int = 3, stride: int = 3) -> np.ndarray: + """MaxPool1d over the time axis of a [channels, time] array.""" + n = 1 + (x.shape[-1] - size) // stride + view = np.lib.stride_tricks.as_strided( + x, + shape=(x.shape[0], n, size), + strides=(x.strides[0], x.strides[1] * stride, x.strides[1]), + ) + return view.max(axis=-1) + + +def conv1d(x: np.ndarray, weight: np.ndarray, bias: np.ndarray, stride: int = 1): + """Valid 1D convolution of [in_ch, time] with [out_ch, in_ch, kernel].""" + in_ch, length = x.shape + out_ch, _, kernel = weight.shape + n = 1 + (length - kernel) // stride + # [in_ch, kernel, n] gather, contracted against the weight in one matmul. + cols = np.lib.stride_tricks.as_strided( + x, + shape=(in_ch, kernel, n), + strides=(x.strides[0], x.strides[1], x.strides[1] * stride), + ).reshape(in_ch * kernel, n) + return weight.reshape(out_ch, in_ch * kernel) @ cols + bias[:, None] + + +def lstm_layer(x, w_ih, w_hh, bias, reverse: bool): + """One direction of a PyTorch LSTM layer over [time, features].""" + steps = x.shape[0] + h = np.zeros(HIDDEN, dtype=np.float32) + c = np.zeros(HIDDEN, dtype=np.float32) + pre = x @ w_ih.T + bias + out = np.empty((steps, HIDDEN), dtype=np.float32) + order = range(steps - 1, -1, -1) if reverse else range(steps) + for t in order: + g = pre[t] + h @ w_hh.T + i = sigmoid(g[0:HIDDEN]) + f = sigmoid(g[HIDDEN : 2 * HIDDEN]) + gg = np.tanh(g[2 * HIDDEN : 3 * HIDDEN]) + o = sigmoid(g[3 * HIDDEN : 4 * HIDDEN]) + c = f * c + i * gg + h = o * np.tanh(c) + out[t] = h + return out + + +def sigmoid(x): + return 1.0 / (1.0 + np.exp(-x)) + + +def log_softmax(x): + m = x.max(axis=-1, keepdims=True) + z = x - m + return z - np.log(np.exp(z).sum(axis=-1, keepdims=True)) + + +class Reference: + def __init__(self, sd: dict): + self.sd = sd + self.sinc = sinc_filters(sd)[:, None, :] + + def sincnet(self, wav: np.ndarray, stages: dict) -> np.ndarray: + sd = self.sd + x = wav[None, :] + x = instance_norm(x, sd["sincnet.wav_norm1d.weight"], sd["sincnet.wav_norm1d.bias"]) + stages["wav_norm"] = x + + x = conv1d(x, self.sinc, np.zeros(SINC_FILTERS, dtype=np.float32), stride=10) + x = np.abs(x) + stages["sinc"] = x + x = max_pool(x) + stages["pool1"] = x + x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.0.weight"], sd["sincnet.norm1d.0.bias"])) + stages["block1"] = x + + x = conv1d(x, sd["sincnet.conv1d.1.weight"], sd["sincnet.conv1d.1.bias"]) + stages["conv2"] = x + x = max_pool(x) + x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.1.weight"], sd["sincnet.norm1d.1.bias"])) + stages["block2"] = x + + x = conv1d(x, sd["sincnet.conv1d.2.weight"], sd["sincnet.conv1d.2.bias"]) + x = max_pool(x) + x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.2.weight"], sd["sincnet.norm1d.2.bias"])) + stages["block3"] = x + return x + + def forward(self, wav: np.ndarray, stages: dict | None = None) -> np.ndarray: + """[num_samples] float32 waveform -> [num_frames, 7] log probabilities.""" + stages = {} if stages is None else stages + sd = self.sd + + x = self.sincnet(wav, stages).T # [time, 60] + + for layer in range(4): + suffix = f"l{layer}" + fwd = lstm_layer( + x, + sd[f"lstm.weight_ih_{suffix}"], + sd[f"lstm.weight_hh_{suffix}"], + sd[f"lstm.bias_ih_{suffix}"] + sd[f"lstm.bias_hh_{suffix}"], + reverse=False, + ) + bwd = lstm_layer( + x, + sd[f"lstm.weight_ih_{suffix}_reverse"], + sd[f"lstm.weight_hh_{suffix}_reverse"], + sd[f"lstm.bias_ih_{suffix}_reverse"] + sd[f"lstm.bias_hh_{suffix}_reverse"], + reverse=True, + ) + x = np.concatenate([fwd, bwd], axis=-1) + stages[f"lstm{layer}"] = x + + for i in range(2): + x = leaky_relu(x @ sd[f"linear.{i}.weight"].T + sd[f"linear.{i}.bias"]) + stages[f"linear{i}"] = x + + logits = x @ sd["classifier.weight"].T + sd["classifier.bias"] + stages["logits"] = logits + return log_softmax(logits) + + +def main() -> int: + import torch + from pyannote.audio import Model + + from model import checkpoint_path + + sd = load_state_dict() + ref = Reference(sd) + + torch_model = Model.from_pretrained(checkpoint_path()) + torch_model.eval() + + rng = np.random.default_rng(0) + worst = 0.0 + for trial in range(3): + wav = rng.standard_normal(NUM_SAMPLES).astype(np.float32) * 0.1 + with torch.no_grad(): + want = torch_model(torch.from_numpy(wav)[None, None, :]).numpy()[0] + got = ref.forward(wav) + assert got.shape == (NUM_FRAMES, 7), got.shape + diff = np.abs(got - want).max() + worst = max(worst, diff) + print(f"trial {trial}: max |log-prob diff| = {diff:.3e}") + + ok = worst <= 2e-4 + print(f"worst {worst:.3e} -> {'PARITY OK' if ok else 'PARITY FAILED'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) |
