diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:03:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:13:28 +0200 |
| commit | 64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch) | |
| tree | 85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /tools | |
| download | pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip | |
Init
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/baseline.py | 137 | ||||
| -rw-r--r-- | tools/embed.py | 45 | ||||
| -rw-r--r-- | tools/export_weights.py | 153 | ||||
| -rw-r--r-- | tools/make_golden.py | 93 | ||||
| -rw-r--r-- | tools/model.py | 137 | ||||
| -rw-r--r-- | tools/pipeline.py | 169 | ||||
| -rw-r--r-- | tools/reference.py | 192 | ||||
| -rw-r--r-- | tools/verify_cli.py | 132 |
8 files changed, 1058 insertions, 0 deletions
diff --git a/tools/baseline.py b/tools/baseline.py new file mode 100644 index 0000000..08ae72a --- /dev/null +++ b/tools/baseline.py @@ -0,0 +1,137 @@ +"""PyTorch baseline measurement and golden-vector capture. + +Usage: + python tools/baseline.py bench [--batch N] + python tools/baseline.py golden <audio.wav> <out.npz> [--chunks N] + +The benchmark protocol is fixed here and mirrored by bench/bench.c so the two +numbers are comparable: 3 warm-up forwards, then 20 timed forwards on the same +input, reporting mean, p50 and p99 in milliseconds plus the realtime factor. +""" + +import argparse +import resource +import statistics +import sys +import time + +import numpy as np + +from model import NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE, checkpoint_path + + +def load_torch_model(): + import torch + from pyannote.audio import Model + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + model.eval() + return model + + +def read_wav(path: str) -> np.ndarray: + import soundfile as sf + + data, rate = sf.read(path, dtype="float32", always_2d=True) + if rate != SAMPLE_RATE: + raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz, got {rate}") + return np.ascontiguousarray(data[:, 0]) + + +def bench(args) -> int: + import torch + + t0 = time.perf_counter() + model = load_torch_model() + init_ms = (time.perf_counter() - t0) * 1e3 + + rng = np.random.default_rng(1) + wav = rng.standard_normal((args.batch, 1, NUM_SAMPLES)).astype(np.float32) * 0.1 + x = torch.from_numpy(wav) + + with torch.no_grad(): + for _ in range(3): + model(x) + times = [] + for _ in range(20): + t = time.perf_counter() + model(x) + times.append((time.perf_counter() - t) * 1e3) + + times.sort() + mean = statistics.fmean(times) + p50 = times[len(times) // 2] + p99 = times[min(len(times) - 1, int(0.99 * len(times)))] + audio_s = 10.0 * args.batch + rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e6 + + print(f"torch batch={args.batch} threads=1") + print(f" init {init_ms:8.1f} ms") + print(f" mean/p50/p99 {mean:8.2f} / {p50:.2f} / {p99:.2f} ms") + print(f" per chunk {mean / args.batch:8.2f} ms") + print(f" realtime factor {audio_s / (mean / 1e3):8.1f}x") + print(f" max rss {rss:8.1f} MB") + return 0 + + +def golden(args) -> int: + import torch + + from reference import Reference + from model import load_state_dict + + wav = read_wav(args.audio) + step = int(1.0 * SAMPLE_RATE) + starts = list(range(0, len(wav) - NUM_SAMPLES + 1, step))[: args.chunks] + if not starts: + raise SystemExit("audio shorter than one 10 s window") + + model = load_torch_model() + inputs = np.stack([wav[s : s + NUM_SAMPLES] for s in starts]) + + outs = [] + with torch.no_grad(): + for i in range(0, len(inputs), 8): + batch = torch.from_numpy(inputs[i : i + 8])[:, None, :] + outs.append(model(batch).numpy()) + outputs = np.concatenate(outs) + assert outputs.shape == (len(starts), NUM_FRAMES, 7) + + # A numpy-reference pass over the same inputs pins the tolerance the C + # engine is later judged against. + ref = Reference(load_state_dict()) + ref_out = np.stack([ref.forward(x) for x in inputs[: min(8, len(inputs))]]) + diff = np.abs(ref_out - outputs[: len(ref_out)]).max() + print(f"numpy reference vs torch on {len(ref_out)} chunks: max diff {diff:.3e}") + + np.savez_compressed( + args.out, + inputs=inputs.astype(np.float32), + outputs=outputs.astype(np.float32), + starts=np.array(starts, dtype=np.int64), + ) + print(f"wrote {args.out}: {len(starts)} chunks from {args.audio}") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + b = sub.add_parser("bench") + b.add_argument("--batch", type=int, default=1) + b.set_defaults(func=bench) + + g = sub.add_parser("golden") + g.add_argument("audio") + g.add_argument("out") + g.add_argument("--chunks", type=int, default=32) + g.set_defaults(func=golden) + + args = ap.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/embed.py b/tools/embed.py new file mode 100644 index 0000000..bc4157e --- /dev/null +++ b/tools/embed.py @@ -0,0 +1,45 @@ +"""Turns the weight blob into a C source file so a binary carries no data file. + +Deliberately depends on nothing but the standard library: this runs inside the +container build, where the checkpoint, PyTorch and pyannote are not present and +should not have to be. Regenerating the blob itself needs all three and is a +separate step. + + python3 tools/embed.py pyseg_weights.bin src/weights_blob.c +""" + +import sys + +HEADER = """/* Generated by tools/embed.py from {name}. Do not edit. + * + * The packed weight blob, so a binary needs no data file at runtime. + * pyseg_create_from_file reads the identical bytes from disk instead. */ + +#include <stddef.h> + +__attribute__((aligned(64))) +const unsigned char pyseg_weights_blob[] = {{ +""" + + +def main(argv: list[str]) -> int: + if len(argv) != 3: + print(__doc__.strip(), file=sys.stderr) + return 2 + + with open(argv[1], "rb") as f: + data = f.read() + + step = 24 + with open(argv[2], "w") as out: + out.write(HEADER.format(name=argv[1])) + for i in range(0, len(data), step): + out.write("".join(f"{b:d}," for b in data[i : i + step])) + out.write("\n") + out.write("};\n") + out.write(f"const size_t pyseg_weights_blob_len = {len(data)};\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/export_weights.py b/tools/export_weights.py new file mode 100644 index 0000000..f25455b --- /dev/null +++ b/tools/export_weights.py @@ -0,0 +1,153 @@ +"""Packs the checkpoint into the layout the C engine reads. + +Everything the engine needs is a matrix of shape [N, K] multiplied against +activation windows of K contiguous floats, so the export repacks convolution +kernels from PyTorch's [out, in, tap] order into [out, tap * in], materialises +the parametric sinc filterbank, and merges the two LSTM bias vectors. Nothing +is transformed at load time. + + python tools/export_weights.py <out-dir> + +writes <out-dir>/pyseg_weights.bin. tools/embed.py turns that into the C +source a self-contained binary compiles in; it is kept separate because it +needs nothing but the standard library and therefore runs anywhere, including +inside a container build. +""" + +import argparse +import struct +import sys +from pathlib import Path + +import numpy as np + +from model import SINC_FILTERS, load_state_dict, sinc_filters + +MAGIC = b"PSG1" +VERSION = 1 +NAME_LEN = 24 +ALIGN = 64 + +# Column count the sinc kernel is padded to. The 251 real taps are followed by +# five zero weights so the kernel's K is a multiple of four and the innermost +# loop needs no tail; the samples those zeros multiply are always in range +# because the last window starts at sample 159740 of 160000. +SINC_K = 256 + + +def fnv1a(data: bytes) -> int: + h = 0x811C9DC5 + for b in data: + h = ((h ^ b) * 0x01000193) & 0xFFFFFFFF + return h + + +class Blob: + def __init__(self): + self.tensors = [] + + def add(self, name: str, array: np.ndarray) -> None: + if len(name) >= NAME_LEN: + raise ValueError(f"name too long: {name}") + data = np.ascontiguousarray(array, dtype=np.float32) + self.tensors.append((name, data)) + + def pack(self) -> bytes: + header = struct.calcsize("<4sIII") + entry = struct.calcsize(f"<{NAME_LEN}sIII") + offset = header + entry * len(self.tensors) + offset = (offset + ALIGN - 1) // ALIGN * ALIGN + + table, payload, cursor = [], [], offset + for name, data in self.tensors: + raw = data.tobytes() + table.append((name, cursor, len(raw), fnv1a(raw))) + payload.append(raw) + cursor += len(raw) + pad = (-cursor) % ALIGN + payload.append(b"\0" * pad) + cursor += pad + + out = bytearray() + out += struct.pack("<4sIII", MAGIC, VERSION, len(self.tensors), cursor) + for name, off, nbytes, digest in table: + out += struct.pack( + f"<{NAME_LEN}sIII", name.encode(), off, nbytes, digest + ) + out += b"\0" * (offset - len(out)) + for chunk in payload: + out += chunk + assert len(out) == cursor + return bytes(out) + + +def repack_conv(weight: np.ndarray) -> np.ndarray: + """[out, in, tap] -> [out, tap * in], matching a window of tap frames. + + Activations are stored time-major, so the taps a kernel needs for one + output step are tap consecutive frames of in channels each: one + contiguous span whose element order is tap-major. + """ + out_ch, in_ch, taps = weight.shape + return np.ascontiguousarray(weight.transpose(0, 2, 1).reshape(out_ch, taps * in_ch)) + + +def build(sd: dict) -> Blob: + blob = Blob() + + blob.add("wavnorm", np.stack([sd["sincnet.wav_norm1d.weight"], sd["sincnet.wav_norm1d.bias"]])) + + sinc = np.zeros((SINC_FILTERS, SINC_K), dtype=np.float32) + sinc[:, : sinc_filters(sd).shape[1]] = sinc_filters(sd) + blob.add("sinc_w", sinc) + + for i, tag in enumerate(("norm1", "norm2", "norm3")): + blob.add(f"{tag}_w", sd[f"sincnet.norm1d.{i}.weight"]) + blob.add(f"{tag}_b", sd[f"sincnet.norm1d.{i}.bias"]) + + for src, tag in ((1, "conv2"), (2, "conv3")): + blob.add(f"{tag}_w", repack_conv(sd[f"sincnet.conv1d.{src}.weight"])) + blob.add(f"{tag}_b", sd[f"sincnet.conv1d.{src}.bias"]) + + for layer in range(4): + for tag, suffix in (("f", ""), ("b", "_reverse")): + key = f"lstm{layer}{tag}" + blob.add(f"{key}_ih", sd[f"lstm.weight_ih_l{layer}{suffix}"]) + blob.add(f"{key}_hh", sd[f"lstm.weight_hh_l{layer}{suffix}"]) + blob.add( + f"{key}_bias", + sd[f"lstm.bias_ih_l{layer}{suffix}"] + sd[f"lstm.bias_hh_l{layer}{suffix}"], + ) + + for i in range(2): + blob.add(f"lin{i}_w", sd[f"linear.{i}.weight"]) + blob.add(f"lin{i}_b", sd[f"linear.{i}.bias"]) + + blob.add("cls_w", sd["classifier.weight"]) + blob.add("cls_b", sd["classifier.bias"]) + return blob + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("outdir", nargs="?", default=".") + args = ap.parse_args() + + outdir = Path(args.outdir) + sd = load_state_dict() + blob = build(sd) + data = blob.pack() + + binpath = outdir / "pyseg_weights.bin" + binpath.write_bytes(data) + + params = sum(t.size for _, t in blob.tensors) + print(f"{binpath}: {len(blob.tensors)} tensors, {params} floats, {len(data) / 1e6:.2f} MB") + for name, t in blob.tensors: + print(f" {name:16s} {str(t.shape):16s}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/make_golden.py b/tools/make_golden.py new file mode 100644 index 0000000..109af1d --- /dev/null +++ b/tools/make_golden.py @@ -0,0 +1,93 @@ +"""Converts the captured references into flat binary files the C tests read. + +Keeping the test fixtures as plain little-endian float32 means the C side needs +no parser and no dependency, and the conversion is the only place the two +representations have to agree. + + python tools/make_golden.py chunks golden.npz build/golden.bin + python tools/make_golden.py slide audio.wav build/slide_golden.bin +""" + +import argparse +import struct +import sys + +import numpy as np + +from model import NUM_CLASSES, NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE + +MAGIC_CHUNKS = b"PSGG" +MAGIC_SLIDE = b"PSGS" + + +def chunks(args) -> int: + """Per-window fixtures: waveform in, class probabilities out. + + The torch reference emits log probabilities. They are exponentiated here + because that is what the engine returns, and because a difference of 2e-3 + in a log probability of -17 is a difference of 4e-8 in the probability + itself: comparing in log space would set the tolerance by the least + significant outputs rather than the ones that decide a frame. + """ + data = np.load(args.npz) + inputs = data["inputs"].astype(np.float32) + probs = np.exp(data["outputs"].astype(np.float64)).astype(np.float32) + n = inputs.shape[0] + assert inputs.shape == (n, NUM_SAMPLES) + assert probs.shape == (n, NUM_FRAMES, NUM_CLASSES) + + with open(args.out, "wb") as f: + f.write(struct.pack("<4sIII", MAGIC_CHUNKS, n, NUM_SAMPLES, NUM_FRAMES)) + f.write(inputs.tobytes()) + f.write(np.ascontiguousarray(probs).tobytes()) + print(f"wrote {args.out}: {n} windows") + return 0 + + +def slide(args) -> int: + """Whole-recording fixture: the aggregated tracks pyannote's Inference + produces, next to the samples that produced them.""" + import soundfile as sf + + from pipeline import aggregate_tracks + + wav, rate = sf.read(args.audio, dtype="float32", always_2d=True) + if rate != SAMPLE_RATE: + raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz, got {rate}") + samples = np.ascontiguousarray(wav[: args.limit, 0]) + + tracks = aggregate_tracks(samples, step=args.step) + with open(args.out, "wb") as f: + f.write( + struct.pack( + "<4sIId", MAGIC_SLIDE, len(samples), tracks.shape[0], args.step + ) + ) + f.write(samples.tobytes()) + f.write(np.ascontiguousarray(tracks, dtype=np.float32).tobytes()) + print(f"wrote {args.out}: {len(samples)} samples, {tracks.shape[0]} frames") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + c = sub.add_parser("chunks") + c.add_argument("npz") + c.add_argument("out") + c.set_defaults(func=chunks) + + s = sub.add_parser("slide") + s.add_argument("audio") + s.add_argument("out") + s.add_argument("--step", type=float, default=1.0) + s.add_argument("--limit", type=int, default=45 * SAMPLE_RATE) + s.set_defaults(func=slide) + + args = ap.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/model.py b/tools/model.py new file mode 100644 index 0000000..b6601e3 --- /dev/null +++ b/tools/model.py @@ -0,0 +1,137 @@ +"""Loading and structural description of pyannote/segmentation-3.0. + +The checkpoint is a PyTorch Lightning file whose pickle references pyannote +classes, so reading it requires pyannote.audio to be importable. Everything +downstream of this module works on plain numpy arrays. +""" + +from pathlib import Path + +import numpy as np + +REPO = "pyannote/segmentation-3.0" +SNAPSHOT = ( + Path.home() + / ".cache/huggingface/hub/models--pyannote--segmentation-3.0/snapshots" +) + +SAMPLE_RATE = 16000 +DURATION = 10.0 +NUM_SAMPLES = 160000 +NUM_FRAMES = 589 +NUM_CLASSES = 7 +HIDDEN = 128 +NUM_LAYERS = 4 + +# SincNet geometry: (kernel_size, stride) for conv, abs, pool, conv, pool, conv, pool. +SINC_KERNEL = 251 +SINC_STRIDE = 10 +SINC_FILTERS = 80 + + +def checkpoint_path() -> Path: + """Path of the single snapshot of the model in the local Hugging Face cache.""" + snaps = sorted(SNAPSHOT.glob("*/pytorch_model.bin")) + if not snaps: + raise FileNotFoundError( + f"{REPO} not found under {SNAPSHOT}; run \"hf download {REPO}\" first" + ) + return snaps[-1] + + +def load_state_dict() -> dict: + """Returns the checkpoint tensors as numpy arrays keyed by parameter name.""" + import torch + + ckpt = torch.load(checkpoint_path(), map_location="cpu", weights_only=False) + return {k: v.numpy().copy() for k, v in ckpt["state_dict"].items()} + + +def sinc_filters(sd: dict) -> np.ndarray: + """Materialises the parametric sinc filterbank as a plain [80, 251] matrix. + + asteroid_filterbanks builds the filters from the learned band edges every + forward pass, but the edges are frozen at inference time, so the result is a + constant that can be folded into the exported weights. + """ + low_hz = sd["sincnet.conv1d.0.filterbank.low_hz_"].astype(np.float64) + band_hz = sd["sincnet.conv1d.0.filterbank.band_hz_"].astype(np.float64) + window = sd["sincnet.conv1d.0.filterbank.window_"].astype(np.float64) + n_ = sd["sincnet.conv1d.0.filterbank.n_"].astype(np.float64) + + min_low_hz, min_band_hz = 50.0, 50.0 + low = min_low_hz + np.abs(low_hz) + high = np.clip(low + min_band_hz + np.abs(band_hz), min_low_hz, SAMPLE_RATE / 2) + band = (high - low)[:, 0] + + ft_low = low @ n_ + ft_high = high @ n_ + + cos_left = ((np.sin(ft_high) - np.sin(ft_low)) / (n_ / 2)) * window + cos = np.concatenate( + [cos_left, 2 * band[:, None], np.flip(cos_left, axis=1)], axis=1 + ) + cos /= 2 * band[:, None] + + sin_left = ((np.cos(ft_low) - np.cos(ft_high)) / (n_ / 2)) * window + sin = np.concatenate( + [sin_left, np.zeros_like(band[:, None]), -np.flip(sin_left, axis=1)], axis=1 + ) + sin /= 2 * band[:, None] + + return np.concatenate([cos, sin], axis=0).astype(np.float32) + + +def num_frames(num_samples: int) -> int: + """Output frame count of the SincNet stack for a given input length.""" + geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)] + n = num_samples + for kernel, stride in geometry: + n = 1 + (n - kernel) // stride + return n + + +def receptive_field() -> tuple[int, int, int]: + """(size, step, start) of the output frame grid, in samples. + + Frame f covers the 991 input samples starting at f * 270. The grid begins + exactly at sample 0 because the centre of the first frame sits half a + receptive field into the signal. + """ + geometry = [(SINC_KERNEL, SINC_STRIDE), (3, 3), (5, 1), (3, 3), (5, 1), (3, 3)] + + def rf_size(frames: int) -> int: + size = frames + for kernel, stride in reversed(geometry): + size = kernel + (size - 1) * stride + return size + + def rf_centre(frame: int) -> int: + centre = frame + for kernel, stride in reversed(geometry): + centre = centre * stride + (kernel - 1) // 2 + return centre + + size = rf_size(1) + step = rf_size(2) - size + start = rf_centre(0) - (size - 1) // 2 + return size, step, start + + +def powerset_mapping(num_classes: int = 3, max_set_size: int = 2) -> np.ndarray: + """[num_powerset_classes, num_classes] membership matrix in pyannote order.""" + from itertools import combinations + + rows = [] + for size in range(max_set_size + 1): + for members in combinations(range(num_classes), size): + row = np.zeros(num_classes, dtype=np.float32) + row[list(members)] = 1.0 + rows.append(row) + return np.stack(rows) + + +FRAME_SIZE, FRAME_STEP, FRAME_START = receptive_field() + +assert num_frames(NUM_SAMPLES) == NUM_FRAMES +assert (FRAME_SIZE, FRAME_STEP, FRAME_START) == (991, 270, 0) diff --git a/tools/pipeline.py b/tools/pipeline.py new file mode 100644 index 0000000..c4f675d --- /dev/null +++ b/tools/pipeline.py @@ -0,0 +1,169 @@ +"""Sliding-window aggregation, the reference for src/slide.c. + +pyannote's Inference cuts a recording into overlapping ten-second windows, +converts each window's powerset output to per-speaker activity, and averages +the windows with a Hamming taper. The taper lets a frame sitting at the edge of +one window, where the LSTM has only seen context on one side, defer to the +window that holds it in the middle. + +Run as a script it checks this implementation against pyannote's own Inference +on real audio: + + python tools/pipeline.py <audio.wav> +""" + +import sys + +import numpy as np + +from model import ( + FRAME_STEP, + NUM_FRAMES, + NUM_SAMPLES, + SAMPLE_RATE, + checkpoint_path, + powerset_mapping, +) + +TRACK_SPEECH, TRACK_OVERLAP, TRACK_COUNT = 0, 1, 2 +NUM_TRACKS = 3 + + +def rint_div(n: int, d: int) -> int: + """Round n/d half to even, matching numpy.rint on the same ratio.""" + q, r = divmod(n, d) + if 2 * r < d: + return q + if 2 * r > d: + return q + 1 + return q + (q & 1) + + +def window_starts(num_samples: int, step_samples: int) -> list[int]: + if num_samples < NUM_SAMPLES: + return [0] + whole = (num_samples - NUM_SAMPLES) // step_samples + 1 + starts = [i * step_samples for i in range(whole)] + if (num_samples - NUM_SAMPLES) % step_samples > 0: + starts.append(whole * step_samples) + return starts + + +def num_aggregated_frames(num_samples: int, step: float) -> int: + step_samples = int(step * SAMPLE_RATE + 0.5) + starts = window_starts(num_samples, step_samples) + frames = rint_div(NUM_SAMPLES + (len(starts) - 1) * step_samples, FRAME_STEP) + 1 + padded = num_samples < NUM_SAMPLES or (num_samples - NUM_SAMPLES) % step_samples > 0 + if padded: + frames = min(frames, num_samples // FRAME_STEP + 1) + return frames + + +def powerset_windows(samples: np.ndarray, step: float, batch: int = 8) -> np.ndarray: + """[num_windows, NUM_FRAMES, 7] class probabilities from the torch model.""" + import torch + from pyannote.audio import Model + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + model.eval() + + step_samples = int(step * SAMPLE_RATE + 0.5) + starts = window_starts(len(samples), step_samples) + windows = np.zeros((len(starts), NUM_SAMPLES), dtype=np.float32) + for i, s in enumerate(starts): + chunk = samples[s : s + NUM_SAMPLES] + windows[i, : len(chunk)] = chunk + + out = [] + with torch.no_grad(): + for i in range(0, len(windows), batch): + logp = model(torch.from_numpy(windows[i : i + batch])[:, None, :]) + out.append(torch.exp(logp).numpy()) + return np.concatenate(out) + + +def window_tracks(probs: np.ndarray) -> np.ndarray: + """[num_windows, NUM_FRAMES, 3] speech / overlap / speaker-count votes. + + The powerset output is decoded by taking the most likely class, which is + the hard decoding pyannote uses, so each window contributes integers rather + than probabilities; the averaging across windows is what makes the + aggregate continuous. + """ + mapping = powerset_mapping() + active = mapping[probs.argmax(axis=-1)] + speakers = active.sum(axis=-1) + return np.stack( + [(speakers >= 1).astype(np.float32), (speakers >= 2).astype(np.float32), speakers], + axis=-1, + ) + + +def overlap_add(votes: np.ndarray, num_samples: int, step: float) -> np.ndarray: + """Hamming-weighted average of per-window tracks onto the frame grid.""" + step_samples = int(step * SAMPLE_RATE + 0.5) + starts = window_starts(num_samples, step_samples) + frames = num_aggregated_frames(num_samples, step) + + taper = np.hamming(NUM_FRAMES).astype(np.float32)[:, None] + total = np.zeros((frames, votes.shape[-1]), dtype=np.float32) + weight = np.zeros((frames, votes.shape[-1]), dtype=np.float32) + + for i, start in enumerate(starts): + base = rint_div(start, FRAME_STEP) + n = min(NUM_FRAMES, frames - base) + if n <= 0: + continue + total[base : base + n] += votes[i, :n] * taper[:n] + weight[base : base + n] += taper[:n] + + return np.where(weight > 1e-12, total / np.maximum(weight, 1e-12), 0.0) + + +def aggregate_tracks(samples: np.ndarray, step: float = 1.0) -> np.ndarray: + return overlap_add( + window_tracks(powerset_windows(samples, step)), len(samples), step + ) + + +def main() -> int: + """Checks the speech track against pyannote's own VoiceActivityDetection + inference, which is the aggregation this file reimplements.""" + import soundfile as sf + import torch + from pyannote.audio import Inference, Model + + path = sys.argv[1] if len(sys.argv) > 1 else "testdata_aepyx.wav" + limit = int(sys.argv[2]) if len(sys.argv) > 2 else 45 * SAMPLE_RATE + + wav, rate = sf.read(path, dtype="float32", always_2d=True) + samples = np.ascontiguousarray(wav[:limit, 0]) + if rate != SAMPLE_RATE: + raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz") + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + inference = Inference( + model, + step=1.0, + pre_aggregation_hook=lambda s: np.max(s, axis=-1, keepdims=True), + ) + want = inference( + {"waveform": torch.from_numpy(samples)[None, :], "sample_rate": SAMPLE_RATE} + ).data[:, 0] + + got = aggregate_tracks(samples)[:, TRACK_SPEECH] + + print(f"frames: pyannote {len(want)}, here {len(got)}") + n = min(len(want), len(got)) + diff = np.abs(want[:n] - got[:n]).max() + print(f"max |speech score diff| over {n} frames: {diff:.3e}") + + ok = len(want) == len(got) and diff <= 1e-5 + print("AGGREGATION OK" if ok else "AGGREGATION MISMATCH") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/reference.py b/tools/reference.py new file mode 100644 index 0000000..1992f7f --- /dev/null +++ b/tools/reference.py @@ -0,0 +1,192 @@ +"""Numpy forward pass for pyannote/segmentation-3.0. + +This is the executable specification the C engine is written against: every +operator appears here in the order and with the exact arithmetic the checkpoint +was trained with. Run it against the torch model to confirm the two agree: + + .venv/bin/python tools/reference.py + +The stage outputs are returned as well, so a failing parity run can be bisected +to a single stage instead of guessed at. +""" + +import sys + +import numpy as np + +from model import ( + HIDDEN, + NUM_FRAMES, + NUM_SAMPLES, + SINC_FILTERS, + load_state_dict, + sinc_filters, +) + +EPS = 1e-5 +LEAKY = 0.01 + + +def instance_norm(x: np.ndarray, weight: np.ndarray, bias: np.ndarray) -> np.ndarray: + """InstanceNorm1d(affine=True) over [channels, time]; variance is biased.""" + mean = x.mean(axis=-1, keepdims=True) + var = x.var(axis=-1, keepdims=True) + return (x - mean) / np.sqrt(var + EPS) * weight[:, None] + bias[:, None] + + +def leaky_relu(x: np.ndarray) -> np.ndarray: + return np.where(x >= 0, x, LEAKY * x) + + +def max_pool(x: np.ndarray, size: int = 3, stride: int = 3) -> np.ndarray: + """MaxPool1d over the time axis of a [channels, time] array.""" + n = 1 + (x.shape[-1] - size) // stride + view = np.lib.stride_tricks.as_strided( + x, + shape=(x.shape[0], n, size), + strides=(x.strides[0], x.strides[1] * stride, x.strides[1]), + ) + return view.max(axis=-1) + + +def conv1d(x: np.ndarray, weight: np.ndarray, bias: np.ndarray, stride: int = 1): + """Valid 1D convolution of [in_ch, time] with [out_ch, in_ch, kernel].""" + in_ch, length = x.shape + out_ch, _, kernel = weight.shape + n = 1 + (length - kernel) // stride + # [in_ch, kernel, n] gather, contracted against the weight in one matmul. + cols = np.lib.stride_tricks.as_strided( + x, + shape=(in_ch, kernel, n), + strides=(x.strides[0], x.strides[1], x.strides[1] * stride), + ).reshape(in_ch * kernel, n) + return weight.reshape(out_ch, in_ch * kernel) @ cols + bias[:, None] + + +def lstm_layer(x, w_ih, w_hh, bias, reverse: bool): + """One direction of a PyTorch LSTM layer over [time, features].""" + steps = x.shape[0] + h = np.zeros(HIDDEN, dtype=np.float32) + c = np.zeros(HIDDEN, dtype=np.float32) + pre = x @ w_ih.T + bias + out = np.empty((steps, HIDDEN), dtype=np.float32) + order = range(steps - 1, -1, -1) if reverse else range(steps) + for t in order: + g = pre[t] + h @ w_hh.T + i = sigmoid(g[0:HIDDEN]) + f = sigmoid(g[HIDDEN : 2 * HIDDEN]) + gg = np.tanh(g[2 * HIDDEN : 3 * HIDDEN]) + o = sigmoid(g[3 * HIDDEN : 4 * HIDDEN]) + c = f * c + i * gg + h = o * np.tanh(c) + out[t] = h + return out + + +def sigmoid(x): + return 1.0 / (1.0 + np.exp(-x)) + + +def log_softmax(x): + m = x.max(axis=-1, keepdims=True) + z = x - m + return z - np.log(np.exp(z).sum(axis=-1, keepdims=True)) + + +class Reference: + def __init__(self, sd: dict): + self.sd = sd + self.sinc = sinc_filters(sd)[:, None, :] + + def sincnet(self, wav: np.ndarray, stages: dict) -> np.ndarray: + sd = self.sd + x = wav[None, :] + x = instance_norm(x, sd["sincnet.wav_norm1d.weight"], sd["sincnet.wav_norm1d.bias"]) + stages["wav_norm"] = x + + x = conv1d(x, self.sinc, np.zeros(SINC_FILTERS, dtype=np.float32), stride=10) + x = np.abs(x) + stages["sinc"] = x + x = max_pool(x) + stages["pool1"] = x + x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.0.weight"], sd["sincnet.norm1d.0.bias"])) + stages["block1"] = x + + x = conv1d(x, sd["sincnet.conv1d.1.weight"], sd["sincnet.conv1d.1.bias"]) + stages["conv2"] = x + x = max_pool(x) + x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.1.weight"], sd["sincnet.norm1d.1.bias"])) + stages["block2"] = x + + x = conv1d(x, sd["sincnet.conv1d.2.weight"], sd["sincnet.conv1d.2.bias"]) + x = max_pool(x) + x = leaky_relu(instance_norm(x, sd["sincnet.norm1d.2.weight"], sd["sincnet.norm1d.2.bias"])) + stages["block3"] = x + return x + + def forward(self, wav: np.ndarray, stages: dict | None = None) -> np.ndarray: + """[num_samples] float32 waveform -> [num_frames, 7] log probabilities.""" + stages = {} if stages is None else stages + sd = self.sd + + x = self.sincnet(wav, stages).T # [time, 60] + + for layer in range(4): + suffix = f"l{layer}" + fwd = lstm_layer( + x, + sd[f"lstm.weight_ih_{suffix}"], + sd[f"lstm.weight_hh_{suffix}"], + sd[f"lstm.bias_ih_{suffix}"] + sd[f"lstm.bias_hh_{suffix}"], + reverse=False, + ) + bwd = lstm_layer( + x, + sd[f"lstm.weight_ih_{suffix}_reverse"], + sd[f"lstm.weight_hh_{suffix}_reverse"], + sd[f"lstm.bias_ih_{suffix}_reverse"] + sd[f"lstm.bias_hh_{suffix}_reverse"], + reverse=True, + ) + x = np.concatenate([fwd, bwd], axis=-1) + stages[f"lstm{layer}"] = x + + for i in range(2): + x = leaky_relu(x @ sd[f"linear.{i}.weight"].T + sd[f"linear.{i}.bias"]) + stages[f"linear{i}"] = x + + logits = x @ sd["classifier.weight"].T + sd["classifier.bias"] + stages["logits"] = logits + return log_softmax(logits) + + +def main() -> int: + import torch + from pyannote.audio import Model + + from model import checkpoint_path + + sd = load_state_dict() + ref = Reference(sd) + + torch_model = Model.from_pretrained(checkpoint_path()) + torch_model.eval() + + rng = np.random.default_rng(0) + worst = 0.0 + for trial in range(3): + wav = rng.standard_normal(NUM_SAMPLES).astype(np.float32) * 0.1 + with torch.no_grad(): + want = torch_model(torch.from_numpy(wav)[None, None, :]).numpy()[0] + got = ref.forward(wav) + assert got.shape == (NUM_FRAMES, 7), got.shape + diff = np.abs(got - want).max() + worst = max(worst, diff) + print(f"trial {trial}: max |log-prob diff| = {diff:.3e}") + + ok = worst <= 2e-4 + print(f"worst {worst:.3e} -> {'PARITY OK' if ok else 'PARITY FAILED'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify_cli.py b/tools/verify_cli.py new file mode 100644 index 0000000..15e7cef --- /dev/null +++ b/tools/verify_cli.py @@ -0,0 +1,132 @@ +"""End-to-end check of the built binary against pyannote's own pipelines. + +The parity tests compare tensors; this compares what a user actually gets. It +derives speech and overlapped-speech segments from pyannote for a recording, +runs pyseg_cli over the same file, and reports both the segment boundaries +and the fraction of the timeline the two disagree about, which is the number +that matters for anything downstream. + + python tools/verify_cli.py <audio.wav> [--limit SECONDS] [--step SECONDS] +""" + +import argparse +import subprocess +import sys + +import numpy as np + +from model import SAMPLE_RATE, checkpoint_path + +BINARY = "./pyseg_cli" + + +def run_cli(path: str, mode: str, step: float) -> list[tuple[float, float]]: + out = subprocess.run( + [BINARY, "--mode", mode, "--step", str(step), path], + check=True, + capture_output=True, + text=True, + ).stdout + return [ + (float(a), float(b)) + for a, b in (line.split() for line in out.splitlines() if line.strip()) + ] + + +def run_pyannote(path: str, mode: str, step: float): + """Reference segments for one track. + + Speech goes through pyannote's own VoiceActivityDetection pipeline. Overlap + is assembled here from the same parts, because the pipeline that used to + wrap them was dropped from pyannote.audio: take the second largest + per-speaker score of each frame, which for this model's hard multi-label + output is one exactly when at least two speakers are active, aggregate, and + binarize with the same thresholds. + """ + import torch + from pyannote.audio import Inference, Model + from pyannote.audio.pipelines import VoiceActivityDetection + from pyannote.audio.utils.signal import Binarize + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + + if mode == "speech": + pipeline = VoiceActivityDetection(segmentation=model, step=step) + pipeline.instantiate({"min_duration_on": 0.0, "min_duration_off": 0.0}) + annotation = pipeline(path) + else: + inference = Inference( + model, + step=step, + pre_aggregation_hook=lambda s: np.partition(s, -2, axis=-1)[:, :, -2, np.newaxis], + ) + binarize = Binarize( + onset=0.5, offset=0.5, min_duration_on=0.0, min_duration_off=0.0 + ) + annotation = binarize(inference(path)) + + return [(s.start, s.end) for s in annotation.get_timeline()] + + +def timeline_mask(segments, duration: float, resolution: float = 0.01) -> np.ndarray: + n = int(duration / resolution) + 1 + mask = np.zeros(n, dtype=bool) + for start, end in segments: + mask[int(start / resolution) : int(end / resolution) + 1] = True + return mask + + +def compare(name: str, got, want, duration: float) -> bool: + a, b = timeline_mask(got, duration), timeline_mask(want, duration) + n = min(len(a), len(b)) + disagree = np.count_nonzero(a[:n] != b[:n]) / n + covered = np.count_nonzero(b[:n]) / n + + print(f"{name}: {len(got)} segments here, {len(want)} from pyannote") + print(f" reference covers {covered:6.2%} of the timeline") + print(f" disagreement {disagree:6.4%} of the timeline") + + worst = 0.0 + for (s0, e0), (s1, e1) in zip(got, want): + worst = max(worst, abs(s0 - s1), abs(e0 - e1)) + if len(got) == len(want): + print(f" worst boundary shift {worst * 1e3:.1f} ms") + + ok = disagree <= 1e-3 + print(f" {'OK' if ok else 'MISMATCH'}") + return ok + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("audio") + ap.add_argument("--limit", type=int, default=None, help="seconds of audio to use") + ap.add_argument("--step", type=float, default=1.0) + args = ap.parse_args() + + import soundfile as sf + + info = sf.info(args.audio) + if info.samplerate != SAMPLE_RATE: + raise SystemExit(f"{args.audio}: expected {SAMPLE_RATE} Hz") + duration = info.duration if args.limit is None else min(info.duration, args.limit) + + path = args.audio + if args.limit is not None: + wav, rate = sf.read(args.audio, dtype="float32", always_2d=True) + path = "build/verify_clip.wav" + sf.write(path, wav[: args.limit * rate, :1], rate, subtype="PCM_16") + + ok = True + for mode in ("speech", "overlap"): + got = run_cli(path, mode, args.step) + want = run_pyannote(path, mode, args.step) + ok &= compare(mode, got, want, duration) + + print("VERIFY OK" if ok else "VERIFY FAILED") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) |
