aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-13 20:03:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-13 20:13:28 +0200
commit64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch)
tree85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /tools
downloadpyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz
pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip
Init
Diffstat (limited to 'tools')
-rw-r--r--tools/baseline.py137
-rw-r--r--tools/embed.py45
-rw-r--r--tools/export_weights.py153
-rw-r--r--tools/make_golden.py93
-rw-r--r--tools/model.py137
-rw-r--r--tools/pipeline.py169
-rw-r--r--tools/reference.py192
-rw-r--r--tools/verify_cli.py132
8 files changed, 1058 insertions, 0 deletions
diff --git a/tools/baseline.py b/tools/baseline.py
new file mode 100644
index 0000000..08ae72a
--- /dev/null
+++ b/tools/baseline.py
@@ -0,0 +1,137 @@
+"""PyTorch baseline measurement and golden-vector capture.
+
+Usage:
+ python tools/baseline.py bench [--batch N]
+ python tools/baseline.py golden <audio.wav> <out.npz> [--chunks N]
+
+The benchmark protocol is fixed here and mirrored by bench/bench.c so the two
+numbers are comparable: 3 warm-up forwards, then 20 timed forwards on the same
+input, reporting mean, p50 and p99 in milliseconds plus the realtime factor.
+"""
+
+import argparse
+import resource
+import statistics
+import sys
+import time
+
+import numpy as np
+
+from model import NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE, checkpoint_path
+
+
+def load_torch_model():
+ import torch
+ from pyannote.audio import Model
+
+ torch.set_num_threads(1)
+ model = Model.from_pretrained(checkpoint_path())
+ model.eval()
+ return model
+
+
+def read_wav(path: str) -> np.ndarray:
+ import soundfile as sf
+
+ data, rate = sf.read(path, dtype="float32", always_2d=True)
+ if rate != SAMPLE_RATE:
+ raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz, got {rate}")
+ return np.ascontiguousarray(data[:, 0])
+
+
+def bench(args) -> int:
+ import torch
+
+ t0 = time.perf_counter()
+ model = load_torch_model()
+ init_ms = (time.perf_counter() - t0) * 1e3
+
+ rng = np.random.default_rng(1)
+ wav = rng.standard_normal((args.batch, 1, NUM_SAMPLES)).astype(np.float32) * 0.1
+ x = torch.from_numpy(wav)
+
+ with torch.no_grad():
+ for _ in range(3):
+ model(x)
+ times = []
+ for _ in range(20):
+ t = time.perf_counter()
+ model(x)
+ times.append((time.perf_counter() - t) * 1e3)
+
+ times.sort()
+ mean = statistics.fmean(times)
+ p50 = times[len(times) // 2]
+ p99 = times[min(len(times) - 1, int(0.99 * len(times)))]
+ audio_s = 10.0 * args.batch
+ rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e6
+
+ print(f"torch batch={args.batch} threads=1")
+ print(f" init {init_ms:8.1f} ms")
+ print(f" mean/p50/p99 {mean:8.2f} / {p50:.2f} / {p99:.2f} ms")
+ print(f" per chunk {mean / args.batch:8.2f} ms")
+ print(f" realtime factor {audio_s / (mean / 1e3):8.1f}x")
+ print(f" max rss {rss:8.1f} MB")
+ return 0
+
+
+def golden(args) -> int:
+ import torch
+
+ from reference import Reference
+ from model import load_state_dict
+
+ wav = read_wav(args.audio)
+ step = int(1.0 * SAMPLE_RATE)
+ starts = list(range(0, len(wav) - NUM_SAMPLES + 1, step))[: args.chunks]
+ if not starts:
+ raise SystemExit("audio shorter than one 10 s window")
+
+ model = load_torch_model()
+ inputs = np.stack([wav[s : s + NUM_SAMPLES] for s in starts])
+
+ outs = []
+ with torch.no_grad():
+ for i in range(0, len(inputs), 8):
+ batch = torch.from_numpy(inputs[i : i + 8])[:, None, :]
+ outs.append(model(batch).numpy())
+ outputs = np.concatenate(outs)
+ assert outputs.shape == (len(starts), NUM_FRAMES, 7)
+
+ # A numpy-reference pass over the same inputs pins the tolerance the C
+ # engine is later judged against.
+ ref = Reference(load_state_dict())
+ ref_out = np.stack([ref.forward(x) for x in inputs[: min(8, len(inputs))]])
+ diff = np.abs(ref_out - outputs[: len(ref_out)]).max()
+ print(f"numpy reference vs torch on {len(ref_out)} chunks: max diff {diff:.3e}")
+
+ np.savez_compressed(
+ args.out,
+ inputs=inputs.astype(np.float32),
+ outputs=outputs.astype(np.float32),
+ starts=np.array(starts, dtype=np.int64),
+ )
+ print(f"wrote {args.out}: {len(starts)} chunks from {args.audio}")
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ sub = ap.add_subparsers(dest="cmd", required=True)
+
+ b = sub.add_parser("bench")
+ b.add_argument("--batch", type=int, default=1)
+ b.set_defaults(func=bench)
+
+ g = sub.add_parser("golden")
+ g.add_argument("audio")
+ g.add_argument("out")
+ g.add_argument("--chunks", type=int, default=32)
+ g.set_defaults(func=golden)
+
+ args = ap.parse_args()
+ return args.func(args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/embed.py b/tools/embed.py
new file mode 100644
index 0000000..bc4157e
--- /dev/null
+++ b/tools/embed.py
@@ -0,0 +1,45 @@
+"""Turns the weight blob into a C source file so a binary carries no data file.
+
+Deliberately depends on nothing but the standard library: this runs inside the
+container build, where the checkpoint, PyTorch and pyannote are not present and
+should not have to be. Regenerating the blob itself needs all three and is a
+separate step.
+
+ python3 tools/embed.py pyseg_weights.bin src/weights_blob.c
+"""
+
+import sys
+
+HEADER = """/* Generated by tools/embed.py from {name}. Do not edit.
+ *
+ * The packed weight blob, so a binary needs no data file at runtime.
+ * pyseg_create_from_file reads the identical bytes from disk instead. */
+
+#include <stddef.h>
+
+__attribute__((aligned(64)))
+const unsigned char pyseg_weights_blob[] = {{
+"""
+
+
+def main(argv: list[str]) -> int:
+ if len(argv) != 3:
+ print(__doc__.strip(), file=sys.stderr)
+ return 2
+
+ with open(argv[1], "rb") as f:
+ data = f.read()
+
+ step = 24
+ with open(argv[2], "w") as out:
+ out.write(HEADER.format(name=argv[1]))
+ for i in range(0, len(data), step):
+ out.write("".join(f"{b:d}," for b in data[i : i + step]))
+ out.write("\n")
+ out.write("};\n")
+ out.write(f"const size_t pyseg_weights_blob_len = {len(data)};\n")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/tools/export_weights.py b/tools/export_weights.py
new file mode 100644
index 0000000..f25455b
--- /dev/null
+++ b/tools/export_weights.py
@@ -0,0 +1,153 @@
+"""Packs the checkpoint into the layout the C engine reads.
+
+Everything the engine needs is a matrix of shape [N, K] multiplied against
+activation windows of K contiguous floats, so the export repacks convolution
+kernels from PyTorch's [out, in, tap] order into [out, tap * in], materialises
+the parametric sinc filterbank, and merges the two LSTM bias vectors. Nothing
+is transformed at load time.
+
+ python tools/export_weights.py <out-dir>
+
+writes <out-dir>/pyseg_weights.bin. tools/embed.py turns that into the C
+source a self-contained binary compiles in; it is kept separate because it
+needs nothing but the standard library and therefore runs anywhere, including
+inside a container build.
+"""
+
+import argparse
+import struct
+import sys
+from pathlib import Path
+
+import numpy as np
+
+from model import SINC_FILTERS, load_state_dict, sinc_filters
+
+MAGIC = b"PSG1"
+VERSION = 1
+NAME_LEN = 24
+ALIGN = 64
+
+# Column count the sinc kernel is padded to. The 251 real taps are followed by
+# five zero weights so the kernel's K is a multiple of four and the innermost
+# loop needs no tail; the samples those zeros multiply are always in range
+# because the last window starts at sample 159740 of 160000.
+SINC_K = 256
+
+
+def fnv1a(data: bytes) -> int:
+ h = 0x811C9DC5
+ for b in data:
+ h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF
+ return h
+
+
+class Blob:
+ def __init__(self):
+ self.tensors = []
+
+ def add(self, name: str, array: np.ndarray) -> None:
+ if len(name) >= NAME_LEN:
+ raise ValueError(f"name too long: {name}")
+ data = np.ascontiguousarray(array, dtype=np.float32)
+ self.tensors.append((name, data))
+
+ def pack(self) -> bytes:
+ header = struct.calcsize("<4sIII")
+ entry = struct.calcsize(f"<{NAME_LEN}sIII")
+ offset = header + entry * len(self.tensors)
+ offset = (offset + ALIGN - 1) // ALIGN * ALIGN
+
+ table, payload, cursor = [], [], offset
+ for name, data in self.tensors:
+ raw = data.tobytes()
+ table.append((name, cursor, len(raw), fnv1a(raw)))
+ payload.append(raw)
+ cursor += len(raw)
+ pad = (-cursor) % ALIGN
+ payload.append(b"\0" * pad)
+ cursor += pad
+
+ out = bytearray()
+ out += struct.pack("<4sIII", MAGIC, VERSION, len(self.tensors), cursor)
+ for name, off, nbytes, digest in table:
+ out += struct.pack(
+ f"<{NAME_LEN}sIII", name.encode(), off, nbytes, digest
+ )
+ out += b"\0" * (offset - len(out))
+ for chunk in payload:
+ out += chunk
+ assert len(out) == cursor
+ return bytes(out)
+
+
+def repack_conv(weight: np.ndarray) -> np.ndarray:
+ """[out, in, tap] -> [out, tap * in], matching a window of tap frames.
+
+ Activations are stored time-major, so the taps a kernel needs for one
+ output step are tap consecutive frames of in channels each: one
+ contiguous span whose element order is tap-major.
+ """
+ out_ch, in_ch, taps = weight.shape
+ return np.ascontiguousarray(weight.transpose(0, 2, 1).reshape(out_ch, taps * in_ch))
+
+
+def build(sd: dict) -> Blob:
+ blob = Blob()
+
+ blob.add("wavnorm", np.stack([sd["sincnet.wav_norm1d.weight"], sd["sincnet.wav_norm1d.bias"]]))
+
+ sinc = np.zeros((SINC_FILTERS, SINC_K), dtype=np.float32)
+ sinc[:, : sinc_filters(sd).shape[1]] = sinc_filters(sd)
+ blob.add("sinc_w", sinc)
+
+ for i, tag in enumerate(("norm1", "norm2", "norm3")):
+ blob.add(f"{tag}_w", sd[f"sincnet.norm1d.{i}.weight"])
+ blob.add(f"{tag}_b", sd[f"sincnet.norm1d.{i}.bias"])
+
+ for src, tag in ((1, "conv2"), (2, "conv3")):
+ blob.add(f"{tag}_w", repack_conv(sd[f"sincnet.conv1d.{src}.weight"]))
+ blob.add(f"{tag}_b", sd[f"sincnet.conv1d.{src}.bias"])
+
+ for layer in range(4):
+ for tag, suffix in (("f", ""), ("b", "_reverse")):
+ key = f"lstm{layer}{tag}"
+ blob.add(f"{key}_ih", sd[f"lstm.weight_ih_l{layer}{suffix}"])
+ blob.add(f"{key}_hh", sd[f"lstm.weight_hh_l{layer}{suffix}"])
+ blob.add(
+ f"{key}_bias",
+ sd[f"lstm.bias_ih_l{layer}{suffix}"] + sd[f"lstm.bias_hh_l{layer}{suffix}"],
+ )
+
+ for i in range(2):
+ blob.add(f"lin{i}_w", sd[f"linear.{i}.weight"])
+ blob.add(f"lin{i}_b", sd[f"linear.{i}.bias"])
+
+ blob.add("cls_w", sd["classifier.weight"])
+ blob.add("cls_b", sd["classifier.bias"])
+ return blob
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("outdir", nargs="?", default=".")
+ args = ap.parse_args()
+
+ outdir = Path(args.outdir)
+ sd = load_state_dict()
+ blob = build(sd)
+ data = blob.pack()
+
+ binpath = outdir / "pyseg_weights.bin"
+ binpath.write_bytes(data)
+
+ params = sum(t.size for _, t in blob.tensors)
+ print(f"{binpath}: {len(blob.tensors)} tensors, {params} floats, {len(data) / 1e6:.2f} MB")
+ for name, t in blob.tensors:
+ print(f" {name:16s} {str(t.shape):16s}")
+
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/make_golden.py b/tools/make_golden.py
new file mode 100644
index 0000000..109af1d
--- /dev/null
+++ b/tools/make_golden.py
@@ -0,0 +1,93 @@
+"""Converts the captured references into flat binary files the C tests read.
+
+Keeping the test fixtures as plain little-endian float32 means the C side needs
+no parser and no dependency, and the conversion is the only place the two
+representations have to agree.
+
+ python tools/make_golden.py chunks golden.npz build/golden.bin
+ python tools/make_golden.py slide audio.wav build/slide_golden.bin
+"""
+
+import argparse
+import struct
+import sys
+
+import numpy as np
+
+from model import NUM_CLASSES, NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE
+
+MAGIC_CHUNKS = b"PSGG"
+MAGIC_SLIDE = b"PSGS"
+
+
+def chunks(args) -> int:
+ """Per-window fixtures: waveform in, class probabilities out.
+
+ The torch reference emits log probabilities. They are exponentiated here
+ because that is what the engine returns, and because a difference of 2e-3
+ in a log probability of -17 is a difference of 4e-8 in the probability
+ itself: comparing in log space would set the tolerance by the least
+ significant outputs rather than the ones that decide a frame.
+ """
+ data = np.load(args.npz)
+ inputs = data["inputs"].astype(np.float32)
+ probs = np.exp(data["outputs"].astype(np.float64)).astype(np.float32)
+ n = inputs.shape[0]
+ assert inputs.shape == (n, NUM_SAMPLES)
+ assert probs.shape == (n, NUM_FRAMES, NUM_CLASSES)
+
+ with open(args.out, "wb") as f:
+ f.write(struct.pack("<4sIII", MAGIC_CHUNKS, n, NUM_SAMPLES, NUM_FRAMES))
+ f.write(inputs.tobytes())
+ f.write(np.ascontiguousarray(probs).tobytes())
+ print(f"wrote {args.out}: {n} windows")
+ return 0
+
+
+def slide(args) -> int:
+ """Whole-recording fixture: the aggregated tracks pyannote's Inference
+ produces, next to the samples that produced them."""
+ import soundfile as sf
+
+ from pipeline import aggregate_tracks
+
+ wav, rate = sf.read(args.audio, dtype="float32", always_2d=True)
+ if rate != SAMPLE_RATE:
+ raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz, got {rate}")
+ samples = np.ascontiguousarray(wav[: args.limit, 0])
+
+ tracks = aggregate_tracks(samples, step=args.step)
+ with open(args.out, "wb") as f:
+ f.write(
+ struct.pack(
+ "<4sIId", MAGIC_SLIDE, len(samples), tracks.shape[0], args.step
+ )
+ )
+ f.write(samples.tobytes())
+ f.write(np.ascontiguousarray(tracks, dtype=np.float32).tobytes())
+ print(f"wrote {args.out}: {len(samples)} samples, {tracks.shape[0]} frames")
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ sub = ap.add_subparsers(dest="cmd", required=True)
+
+ c = sub.add_parser("chunks")
+ c.add_argument("npz")
+ c.add_argument("out")
+ c.set_defaults(func=chunks)
+
+ s = sub.add_parser("slide")
+ s.add_argument("audio")
+ s.add_argument("out")
+ s.add_argument("--step", type=float, default=1.0)
+ s.add_argument("--limit", type=int, default=45 * SAMPLE_RATE)
+ s.set_defaults(func=slide)
+
+ args = ap.parse_args()
+ return args.func(args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/model.py b/tools/model.py
new file mode 100644
index 0000000..b6601e3
--- /dev/null
+++ b/tools/model.py
@@ -0,0 +1,137 @@
+"""Loading and structural description of pyannote/segmentation-3.0.
+
+The checkpoint is a PyTorch Lightning file whose pickle references pyannote
+classes, so reading it requires pyannote.audio to be importable. Everything
+downstream of this module works on plain numpy arrays.
+"""
+
+from pathlib import Path
+
+import numpy as np
+
+REPO = "pyannote/segmentation-3.0"
+SNAPSHOT = (
+ Path.home()
+ / ".cache/huggingface/hub/models--pyannote--segmentation-3.0/snapshots"
+)
+
+SAMPLE_RATE = 16000
+DURATION = 10.0
+NUM_SAMPLES = 160000
+NUM_FRAMES = 589
+NUM_CLASSES = 7
+HIDDEN = 128
+NUM_LAYERS = 4
+
+# SincNet geometry: (kernel_size, stride) for conv, abs, pool, conv, pool, conv, pool.
+SINC_KERNEL = 251
+SINC_STRIDE = 10
+SINC_FILTERS = 80
+
+
+def checkpoint_path() -> Path:
+ """Path of the single snapshot of the model in the local Hugging Face cache."""
+ snaps = sorted(SNAPSHOT.glob("*/pytorch_model.bin"))
+ if not snaps:
+ raise FileNotFoundError(
+ f"{REPO} not found under {SNAPSHOT}; run \"hf download {REPO}\" first"
+ )
+ return snaps[-1]
+
+
+def load_state_dict() -> dict:
+ """Returns the checkpoint tensors as numpy arrays keyed by parameter name."""
+ import torch
+
+ ckpt = torch.load(checkpoint_path(), map_location="cpu", weights_only=False)
+ return {k: v.numpy().copy() for k, v in ckpt["state_dict"].items()}
+
+
+def sinc_filters(sd: dict) -> np.ndarray:
+ """Materialises the parametric sinc filterbank as a plain [80, 251] matrix.
+
+ asteroid_filterbanks builds the filters from the learned band edges every
+ forward pass, but the edges are frozen at inference time, so the result is a
+ constant that can be folded into the exported weights.
+ """
+ low_hz = sd["sincnet.conv1d.0.filterbank.low_hz_"].astype(np.float64)
+ band_hz = sd["sincnet.conv1d.0.filterbank.band_hz_"].astype(np.float64)
+ window = sd["sincnet.conv1d.0.filterbank.window_"].astype(np.float64)
+ n_ = sd["sincnet.conv1d.0.filterbank.n_"].astype(np.float64)
+
+ min_low_hz, min_band_hz = 50.0, 50.0
+ low = min_low_hz + np.abs(low_hz)
+ high = np.clip(low + min_band_hz + np.abs(band_hz), min_low_hz, SAMPLE_RATE / 2)
+ band = (high - low)[:, 0]
+
+ ft_low = low @ n_
+ ft_high = high @ n_
+
+ cos_left = ((np.sin(ft_high) - np.sin(ft_low)) / (n_ / 2)) * window
+ cos = np.concatenate(
+ [cos_left, 2 * band[:, None], np.flip(cos_left, axis=1)], axis=1
+ )
+ cos /= 2 * band[:, None]
+
+ sin_left = ((np.cos(ft_low) - np.cos(ft_high)) / (n_ / 2)) * window
+ sin = np.concatenate(
+ [sin_left, np.zeros_like(band[:, None]), -np.flip(sin_left, axis=1)], axis=1
+ )
+ sin /= 2 * band[:, None]
+
+ return np.concatenate([cos, sin], axis=0).astype(np.float32)
+
+
+def num_frames(num_samples: int) -> int:
+ """Output frame count of the SincNet stack for a given input length."""
+ geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)]
+ n = num_samples
+ for kernel, stride in geometry:
+ n = 1 + (n - kernel) // stride
+ return n
+
+
+def receptive_field() -> tuple[int, int, int]:
+ """(size, step, start) of the output frame grid, in samples.
+
+ Frame f covers the 991 input samples starting at f * 270. The grid begins
+ exactly at sample 0 because the centre of the first frame sits half a
+ receptive field into the signal.
+ """
+ geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)]
+
+ def rf_size(frames: int) -> int:
+ size = frames
+ for kernel, stride in reversed(geometry):
+ size = kernel + (size - 1) * stride
+ return size
+
+ def rf_centre(frame: int) -> int:
+ centre = frame
+ for kernel, stride in reversed(geometry):
+ centre = centre * stride + (kernel - 1) // 2
+ return centre
+
+ size = rf_size(1)
+ step = rf_size(2) - size
+ start = rf_centre(0) - (size - 1) // 2
+ return size, step, start
+
+
+def powerset_mapping(num_classes: int = 3, max_set_size: int = 2) -> np.ndarray:
+ """[num_powerset_classes, num_classes] membership matrix in pyannote order."""
+ from itertools import combinations
+
+ rows = []
+ for size in range(max_set_size + 1):
+ for members in combinations(range(num_classes), size):
+ row = np.zeros(num_classes, dtype=np.float32)
+ row[list(members)] = 1.0
+ rows.append(row)
+ return np.stack(rows)
+
+
+FRAME_SIZE, FRAME_STEP, FRAME_START = receptive_field()
+
+assert num_frames(NUM_SAMPLES) == NUM_FRAMES
+assert (FRAME_SIZE, FRAME_STEP, FRAME_START) == (991, 270, 0)
diff --git a/tools/pipeline.py b/tools/pipeline.py
new file mode 100644
index 0000000..c4f675d
--- /dev/null
+++ b/tools/pipeline.py
@@ -0,0 +1,169 @@
+"""Sliding-window aggregation, the reference for src/slide.c.
+
+pyannote's Inference cuts a recording into overlapping ten-second windows,
+converts each window's powerset output to per-speaker activity, and averages
+the windows with a Hamming taper. The taper lets a frame sitting at the edge of
+one window, where the LSTM has only seen context on one side, defer to the
+window that holds it in the middle.
+
+Run as a script it checks this implementation against pyannote's own Inference
+on real audio:
+
+ python tools/pipeline.py <audio.wav>
+"""
+
+import sys
+
+import numpy as np
+
+from model import (
+ FRAME_STEP,
+ NUM_FRAMES,
+ NUM_SAMPLES,
+ SAMPLE_RATE,
+ checkpoint_path,
+ powerset_mapping,
+)
+
+TRACK_SPEECH, TRACK_OVERLAP, TRACK_COUNT = 0, 1, 2
+NUM_TRACKS = 3
+
+
+def rint_div(n: int, d: int) -> int:
+ """Round n/d half to even, matching numpy.rint on the same ratio."""
+ q, r = divmod(n, d)
+ if 2 * r < d:
+ return q
+ if 2 * r > d:
+ return q + 1
+ return q + (q & 1)
+
+
+def window_starts(num_samples: int, step_samples: int) -> list[int]:
+ if num_samples < NUM_SAMPLES:
+ return [0]
+ whole = (num_samples - NUM_SAMPLES) // step_samples + 1
+ starts = [i * step_samples for i in range(whole)]
+ if (num_samples - NUM_SAMPLES) % step_samples > 0:
+ starts.append(whole * step_samples)
+ return starts
+
+
+def num_aggregated_frames(num_samples: int, step: float) -> int:
+ step_samples = int(step * SAMPLE_RATE + 0.5)
+ starts = window_starts(num_samples, step_samples)
+ frames = rint_div(NUM_SAMPLES + (len(starts) - 1) * step_samples, FRAME_STEP) + 1
+ padded = num_samples < NUM_SAMPLES or (num_samples - NUM_SAMPLES) % step_samples > 0
+ if padded:
+ frames = min(frames, num_samples // FRAME_STEP + 1)
+ return frames
+
+
+def powerset_windows(samples: np.ndarray, step: float, batch: int = 8) -> np.ndarray:
+ """[num_windows, NUM_FRAMES, 7] class probabilities from the torch model."""
+ import torch
+ from pyannote.audio import Model
+
+ torch.set_num_threads(1)
+ model = Model.from_pretrained(checkpoint_path())
+ model.eval()
+
+ step_samples = int(step * SAMPLE_RATE + 0.5)
+ starts = window_starts(len(samples), step_samples)
+ windows = np.zeros((len(starts), NUM_SAMPLES), dtype=np.float32)
+ for i, s in enumerate(starts):
+ chunk = samples[s : s + NUM_SAMPLES]
+ windows[i, : len(chunk)] = chunk
+
+ out = []
+ with torch.no_grad():
+ for i in range(0, len(windows), batch):
+ logp = model(torch.from_numpy(windows[i : i + batch])[:, None, :])
+ out.append(torch.exp(logp).numpy())
+ return np.concatenate(out)
+
+
+def window_tracks(probs: np.ndarray) -> np.ndarray:
+ """[num_windows, NUM_FRAMES, 3] speech / overlap / speaker-count votes.
+
+ The powerset output is decoded by taking the most likely class, which is
+ the hard decoding pyannote uses, so each window contributes integers rather
+ than probabilities; the averaging across windows is what makes the
+ aggregate continuous.
+ """
+ mapping = powerset_mapping()
+ active = mapping[probs.argmax(axis=-1)]
+ speakers = active.sum(axis=-1)
+ return np.stack(
+ [(speakers >= 1).astype(np.float32), (speakers >= 2).astype(np.float32), speakers],
+ axis=-1,
+ )
+
+
+def overlap_add(votes: np.ndarray, num_samples: int, step: float) -> np.ndarray:
+ """Hamming-weighted average of per-window tracks onto the frame grid."""
+ step_samples = int(step * SAMPLE_RATE + 0.5)
+ starts = window_starts(num_samples, step_samples)
+ frames = num_aggregated_frames(num_samples, step)
+
+ taper = np.hamming(NUM_FRAMES).astype(np.float32)[:, None]
+ total = np.zeros((frames, votes.shape[-1]), dtype=np.float32)
+ weight = np.zeros((frames, votes.shape[-1]), dtype=np.float32)
+
+ for i, start in enumerate(starts):
+ base = rint_div(start, FRAME_STEP)
+ n = min(NUM_FRAMES, frames - base)
+ if n <= 0:
+ continue
+ total[base : base + n] += votes[i, :n] * taper[:n]
+ weight[base : base + n] += taper[:n]
+
+ return np.where(weight > 1e-12, total / np.maximum(weight, 1e-12), 0.0)
+
+
+def aggregate_tracks(samples: np.ndarray, step: float = 1.0) -> np.ndarray:
+ return overlap_add(
+ window_tracks(powerset_windows(samples, step)), len(samples), step
+ )
+
+
+def main() -> int:
+ """Checks the speech track against pyannote's own VoiceActivityDetection
+ inference, which is the aggregation this file reimplements."""
+ import soundfile as sf
+ import torch
+ from pyannote.audio import Inference, Model
+
+ path = sys.argv[1] if len(sys.argv) > 1 else "testdata_aepyx.wav"
+ limit = int(sys.argv[2]) if len(sys.argv) > 2 else 45 * SAMPLE_RATE
+
+ wav, rate = sf.read(path, dtype="float32", always_2d=True)
+ samples = np.ascontiguousarray(wav[:limit, 0])
+ if rate != SAMPLE_RATE:
+ raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz")
+
+ torch.set_num_threads(1)
+ model = Model.from_pretrained(checkpoint_path())
+ inference = Inference(
+ model,
+ step=1.0,
+ pre_aggregation_hook=lambda s: np.max(s, axis=-1, keepdims=True),
+ )
+ want = inference(
+ {"waveform": torch.from_numpy(samples)[None, :], "sample_rate": SAMPLE_RATE}
+ ).data[:, 0]
+
+ got = aggregate_tracks(samples)[:, TRACK_SPEECH]
+
+ print(f"frames: pyannote {len(want)}, here {len(got)}")
+ n = min(len(want), len(got))
+ diff = np.abs(want[:n] - got[:n]).max()
+ print(f"max |speech score diff| over {n} frames: {diff:.3e}")
+
+ ok = len(want) == len(got) and diff <= 1e-5
+ print("AGGREGATION OK" if ok else "AGGREGATION MISMATCH")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/reference.py b/tools/reference.py
new file mode 100644
index 0000000..1992f7f
--- /dev/null
+++ b/tools/reference.py
@@ -0,0 +1,192 @@
+"""Numpy forward pass for pyannote/segmentation-3.0.
+
+This is the executable specification the C engine is written against: every
+operator appears here in the order and with the exact arithmetic the checkpoint
+was trained with. Run it against the torch model to confirm the two agree:
+
+ .venv/bin/python tools/reference.py
+
+The stage outputs are returned as well, so a failing parity run can be bisected
+to a single stage instead of guessed at.
+"""
+
+import sys
+
+import numpy as np
+
+from model import (
+ HIDDEN,
+ NUM_FRAMES,
+ NUM_SAMPLES,
+ SINC_FILTERS,
+ load_state_dict,
+ sinc_filters,
+)
+
+EPS = 1e-5
+LEAKY = 0.01
+
+
+def instance_norm(x: np.ndarray, weight: np.ndarray, bias: np.ndarray) -> np.ndarray:
+ """InstanceNorm1d(affine=True) over [channels, time]; variance is biased."""
+ mean = x.mean(axis=-1, keepdims=True)
+ var = x.var(axis=-1, keepdims=True)
+ return (x - mean) / np.sqrt(var + EPS) * weight[:, None] + bias[:, None]
+
+
+def leaky_relu(x: np.ndarray) -> np.ndarray:
+ return np.where(x >= 0, x, LEAKY * x)
+
+
+def max_pool(x: np.ndarray, size: int = 3, stride: int = 3) -> np.ndarray:
+ """MaxPool1d over the time axis of a [channels, time] array."""
+ n = 1 + (x.shape[-1] - size) // stride
+ view = np.lib.stride_tricks.as_strided(
+ x,
+ shape=(x.shape[0], n, size),
+ strides=(x.strides[0], x.strides[1] * stride, x.strides[1]),
+ )
+ return view.max(axis=-1)
+
+
+def conv1d(x: np.ndarray, weight: np.ndarray, bias: np.ndarray, stride: int = 1):
+ """Valid 1D convolution of [in_ch, time] with [out_ch, in_ch, kernel]."""
+ in_ch, length = x.shape
+ out_ch, _, kernel = weight.shape
+ n = 1 + (length - kernel) // stride
+ # [in_ch, kernel, n] gather, contracted against the weight in one matmul.
+ cols = np.lib.stride_tricks.as_strided(
+ x,
+ shape=(in_ch, kernel, n),
+ strides=(x.strides[0], x.strides[1], x.strides[1] * stride),
+ ).reshape(in_ch * kernel, n)
+ return weight.reshape(out_ch, in_ch * kernel) @ cols + bias[:, None]
+
+
+def lstm_layer(x, w_ih, w_hh, bias, reverse: bool):
+ """One direction of a PyTorch LSTM layer over [time, features]."""
+ steps = x.shape[0]
+ h = np.zeros(HIDDEN, dtype=np.float32)
+ c = np.zeros(HIDDEN, dtype=np.float32)
+ pre = x @ w_ih.T + bias
+ out = np.empty((steps, HIDDEN), dtype=np.float32)
+ order = range(steps - 1, -1, -1) if reverse else range(steps)
+ for t in order:
+ g = pre[t] + h @ w_hh.T
+ i = sigmoid(g[0:HIDDEN])
+ f = sigmoid(g[HIDDEN : 2 * HIDDEN])
+ gg = np.tanh(g[2 * HIDDEN : 3 * HIDDEN])
+ o = sigmoid(g[3 * HIDDEN : 4 * HIDDEN])
+ c = f * c + i * gg
+ h = o * np.tanh(c)
+ out[t] = h
+ return out
+
+
+def sigmoid(x):
+ return 1.0 / (1.0 + np.exp(-x))
+
+
+def log_softmax(x):
+ m = x.max(axis=-1, keepdims=True)
+ z = x - m
+ return z - np.log(np.exp(z).sum(axis=-1, keepdims=True))
+
+
+class Reference:
+ def __init__(self, sd: dict):
+ self.sd = sd
+ self.sinc = sinc_filters(sd)[:, None, :]
+
+ def sincnet(self, wav: np.ndarray, stages: dict) -> np.ndarray:
+ sd = self.sd
+ x = wav[None, :]
+ x = instance_norm(x, sd["sincnet.wav_norm1d.weight"], sd["sincnet.wav_norm1d.bias"])
+ stages["wav_norm"] = x
+
+ x = conv1d(x, self.sinc, np.zeros(SINC_FILTERS, dtype=np.float32), stride=10)
+ x = np.abs(x)
+ stages["sinc"] = x
+ x = max_pool(x)
+ stages["pool1"] = x
+ x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.0.weight"], sd["sincnet.norm1d.0.bias"]))
+ stages["block1"] = x
+
+ x = conv1d(x, sd["sincnet.conv1d.1.weight"], sd["sincnet.conv1d.1.bias"])
+ stages["conv2"] = x
+ x = max_pool(x)
+ x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.1.weight"], sd["sincnet.norm1d.1.bias"]))
+ stages["block2"] = x
+
+ x = conv1d(x, sd["sincnet.conv1d.2.weight"], sd["sincnet.conv1d.2.bias"])
+ x = max_pool(x)
+ x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.2.weight"], sd["sincnet.norm1d.2.bias"]))
+ stages["block3"] = x
+ return x
+
+ def forward(self, wav: np.ndarray, stages: dict | None = None) -> np.ndarray:
+ """[num_samples] float32 waveform -> [num_frames, 7] log probabilities."""
+ stages = {} if stages is None else stages
+ sd = self.sd
+
+ x = self.sincnet(wav, stages).T # [time, 60]
+
+ for layer in range(4):
+ suffix = f"l{layer}"
+ fwd = lstm_layer(
+ x,
+ sd[f"lstm.weight_ih_{suffix}"],
+ sd[f"lstm.weight_hh_{suffix}"],
+ sd[f"lstm.bias_ih_{suffix}"] + sd[f"lstm.bias_hh_{suffix}"],
+ reverse=False,
+ )
+ bwd = lstm_layer(
+ x,
+ sd[f"lstm.weight_ih_{suffix}_reverse"],
+ sd[f"lstm.weight_hh_{suffix}_reverse"],
+ sd[f"lstm.bias_ih_{suffix}_reverse"] + sd[f"lstm.bias_hh_{suffix}_reverse"],
+ reverse=True,
+ )
+ x = np.concatenate([fwd, bwd], axis=-1)
+ stages[f"lstm{layer}"] = x
+
+ for i in range(2):
+ x = leaky_relu(x @ sd[f"linear.{i}.weight"].T + sd[f"linear.{i}.bias"])
+ stages[f"linear{i}"] = x
+
+ logits = x @ sd["classifier.weight"].T + sd["classifier.bias"]
+ stages["logits"] = logits
+ return log_softmax(logits)
+
+
+def main() -> int:
+ import torch
+ from pyannote.audio import Model
+
+ from model import checkpoint_path
+
+ sd = load_state_dict()
+ ref = Reference(sd)
+
+ torch_model = Model.from_pretrained(checkpoint_path())
+ torch_model.eval()
+
+ rng = np.random.default_rng(0)
+ worst = 0.0
+ for trial in range(3):
+ wav = rng.standard_normal(NUM_SAMPLES).astype(np.float32) * 0.1
+ with torch.no_grad():
+ want = torch_model(torch.from_numpy(wav)[None, None, :]).numpy()[0]
+ got = ref.forward(wav)
+ assert got.shape == (NUM_FRAMES, 7), got.shape
+ diff = np.abs(got - want).max()
+ worst = max(worst, diff)
+ print(f"trial {trial}: max |log-prob diff| = {diff:.3e}")
+
+ ok = worst <= 2e-4
+ print(f"worst {worst:.3e} -> {'PARITY OK' if ok else 'PARITY FAILED'}")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tools/verify_cli.py b/tools/verify_cli.py
new file mode 100644
index 0000000..15e7cef
--- /dev/null
+++ b/tools/verify_cli.py
@@ -0,0 +1,132 @@
+"""End-to-end check of the built binary against pyannote's own pipelines.
+
+The parity tests compare tensors; this compares what a user actually gets. It
+derives speech and overlapped-speech segments from pyannote for a recording,
+runs pyseg_cli over the same file, and reports both the segment boundaries
+and the fraction of the timeline the two disagree about, which is the number
+that matters for anything downstream.
+
+ python tools/verify_cli.py <audio.wav> [--limit SECONDS] [--step SECONDS]
+"""
+
+import argparse
+import subprocess
+import sys
+
+import numpy as np
+
+from model import SAMPLE_RATE, checkpoint_path
+
+BINARY = "./pyseg_cli"
+
+
+def run_cli(path: str, mode: str, step: float) -> list[tuple[float, float]]:
+ out = subprocess.run(
+ [BINARY, "--mode", mode, "--step", str(step), path],
+ check=True,
+ capture_output=True,
+ text=True,
+ ).stdout
+ return [
+ (float(a), float(b))
+ for a, b in (line.split() for line in out.splitlines() if line.strip())
+ ]
+
+
+def run_pyannote(path: str, mode: str, step: float):
+ """Reference segments for one track.
+
+ Speech goes through pyannote's own VoiceActivityDetection pipeline. Overlap
+ is assembled here from the same parts, because the pipeline that used to
+ wrap them was dropped from pyannote.audio: take the second largest
+ per-speaker score of each frame, which for this model's hard multi-label
+ output is one exactly when at least two speakers are active, aggregate, and
+ binarize with the same thresholds.
+ """
+ import torch
+ from pyannote.audio import Inference, Model
+ from pyannote.audio.pipelines import VoiceActivityDetection
+ from pyannote.audio.utils.signal import Binarize
+
+ torch.set_num_threads(1)
+ model = Model.from_pretrained(checkpoint_path())
+
+ if mode == "speech":
+ pipeline = VoiceActivityDetection(segmentation=model, step=step)
+ pipeline.instantiate({"min_duration_on": 0.0, "min_duration_off": 0.0})
+ annotation = pipeline(path)
+ else:
+ inference = Inference(
+ model,
+ step=step,
+ pre_aggregation_hook=lambda s: np.partition(s, -2, axis=-1)[:, :, -2, np.newaxis],
+ )
+ binarize = Binarize(
+ onset=0.5, offset=0.5, min_duration_on=0.0, min_duration_off=0.0
+ )
+ annotation = binarize(inference(path))
+
+ return [(s.start, s.end) for s in annotation.get_timeline()]
+
+
+def timeline_mask(segments, duration: float, resolution: float = 0.01) -> np.ndarray:
+ n = int(duration / resolution) + 1
+ mask = np.zeros(n, dtype=bool)
+ for start, end in segments:
+ mask[int(start / resolution) : int(end / resolution) + 1] = True
+ return mask
+
+
+def compare(name: str, got, want, duration: float) -> bool:
+ a, b = timeline_mask(got, duration), timeline_mask(want, duration)
+ n = min(len(a), len(b))
+ disagree = np.count_nonzero(a[:n] != b[:n]) / n
+ covered = np.count_nonzero(b[:n]) / n
+
+ print(f"{name}: {len(got)} segments here, {len(want)} from pyannote")
+ print(f" reference covers {covered:6.2%} of the timeline")
+ print(f" disagreement {disagree:6.4%} of the timeline")
+
+ worst = 0.0
+ for (s0, e0), (s1, e1) in zip(got, want):
+ worst = max(worst, abs(s0 - s1), abs(e0 - e1))
+ if len(got) == len(want):
+ print(f" worst boundary shift {worst * 1e3:.1f} ms")
+
+ ok = disagree <= 1e-3
+ print(f" {'OK' if ok else 'MISMATCH'}")
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("audio")
+ ap.add_argument("--limit", type=int, default=None, help="seconds of audio to use")
+ ap.add_argument("--step", type=float, default=1.0)
+ args = ap.parse_args()
+
+ import soundfile as sf
+
+ info = sf.info(args.audio)
+ if info.samplerate != SAMPLE_RATE:
+ raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz")
+ duration = info.duration if args.limit is None else min(info.duration, args.limit)
+
+ path = args.audio
+ if args.limit is not None:
+ wav, rate = sf.read(args.audio, dtype="float32", always_2d=True)
+ path = "build/verify_clip.wav"
+ sf.write(path, wav[: args.limit * rate, :1], rate, subtype="PCM_16")
+
+ ok = True
+ for mode in ("speech", "overlap"):
+ got = run_cli(path, mode, args.step)
+ want = run_pyannote(path, mode, args.step)
+ ok &= compare(mode, got, want, duration)
+
+ print("VERIFY OK" if ok else "VERIFY FAILED")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())