aboutsummaryrefslogtreecommitdiff
path: root/tools/baseline.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-13 20:03:39 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-13 20:13:28 +0200
commit64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch)
tree85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /tools/baseline.py
downloadpyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz
pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip
Init
Diffstat (limited to 'tools/baseline.py')
-rw-r--r--tools/baseline.py137
1 files changed, 137 insertions, 0 deletions
diff --git a/tools/baseline.py b/tools/baseline.py
new file mode 100644
index 0000000..08ae72a
--- /dev/null
+++ b/tools/baseline.py
@@ -0,0 +1,137 @@
+"""PyTorch baseline measurement and golden-vector capture.
+
+Usage:
+ python tools/baseline.py bench [--batch N]
+ python tools/baseline.py golden <audio.wav> <out.npz> [--chunks N]
+
+The benchmark protocol is fixed here and mirrored by bench/bench.c so the two
+numbers are comparable: 3 warm-up forwards, then 20 timed forwards on the same
+input, reporting mean, p50 and p99 in milliseconds plus the realtime factor.
+"""
+
+import argparse
+import resource
+import statistics
+import sys
+import time
+
+import numpy as np
+
+from model import NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE, checkpoint_path
+
+
+def load_torch_model():
+ import torch
+ from pyannote.audio import Model
+
+ torch.set_num_threads(1)
+ model = Model.from_pretrained(checkpoint_path())
+ model.eval()
+ return model
+
+
+def read_wav(path: str) -> np.ndarray:
+ import soundfile as sf
+
+ data, rate = sf.read(path, dtype="float32", always_2d=True)
+ if rate != SAMPLE_RATE:
+ raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz, got {rate}")
+ return np.ascontiguousarray(data[:, 0])
+
+
+def bench(args) -> int:
+ import torch
+
+ t0 = time.perf_counter()
+ model = load_torch_model()
+ init_ms = (time.perf_counter() - t0) * 1e3
+
+ rng = np.random.default_rng(1)
+ wav = rng.standard_normal((args.batch, 1, NUM_SAMPLES)).astype(np.float32) * 0.1
+ x = torch.from_numpy(wav)
+
+ with torch.no_grad():
+ for _ in range(3):
+ model(x)
+ times = []
+ for _ in range(20):
+ t = time.perf_counter()
+ model(x)
+ times.append((time.perf_counter() - t) * 1e3)
+
+ times.sort()
+ mean = statistics.fmean(times)
+ p50 = times[len(times) // 2]
+ p99 = times[min(len(times) - 1, int(0.99 * len(times)))]
+ audio_s = 10.0 * args.batch
+ rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e6
+
+ print(f"torch batch={args.batch} threads=1")
+ print(f" init {init_ms:8.1f} ms")
+ print(f" mean/p50/p99 {mean:8.2f} / {p50:.2f} / {p99:.2f} ms")
+ print(f" per chunk {mean / args.batch:8.2f} ms")
+ print(f" realtime factor {audio_s / (mean / 1e3):8.1f}x")
+ print(f" max rss {rss:8.1f} MB")
+ return 0
+
+
+def golden(args) -> int:
+ import torch
+
+ from reference import Reference
+ from model import load_state_dict
+
+ wav = read_wav(args.audio)
+ step = int(1.0 * SAMPLE_RATE)
+ starts = list(range(0, len(wav) - NUM_SAMPLES + 1, step))[: args.chunks]
+ if not starts:
+ raise SystemExit("audio shorter than one 10 s window")
+
+ model = load_torch_model()
+ inputs = np.stack([wav[s : s + NUM_SAMPLES] for s in starts])
+
+ outs = []
+ with torch.no_grad():
+ for i in range(0, len(inputs), 8):
+ batch = torch.from_numpy(inputs[i : i + 8])[:, None, :]
+ outs.append(model(batch).numpy())
+ outputs = np.concatenate(outs)
+ assert outputs.shape == (len(starts), NUM_FRAMES, 7)
+
+ # A numpy-reference pass over the same inputs pins the tolerance the C
+ # engine is later judged against.
+ ref = Reference(load_state_dict())
+ ref_out = np.stack([ref.forward(x) for x in inputs[: min(8, len(inputs))]])
+ diff = np.abs(ref_out - outputs[: len(ref_out)]).max()
+ print(f"numpy reference vs torch on {len(ref_out)} chunks: max diff {diff:.3e}")
+
+ np.savez_compressed(
+ args.out,
+ inputs=inputs.astype(np.float32),
+ outputs=outputs.astype(np.float32),
+ starts=np.array(starts, dtype=np.int64),
+ )
+ print(f"wrote {args.out}: {len(starts)} chunks from {args.audio}")
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ sub = ap.add_subparsers(dest="cmd", required=True)
+
+ b = sub.add_parser("bench")
+ b.add_argument("--batch", type=int, default=1)
+ b.set_defaults(func=bench)
+
+ g = sub.add_parser("golden")
+ g.add_argument("audio")
+ g.add_argument("out")
+ g.add_argument("--chunks", type=int, default=32)
+ g.set_defaults(func=golden)
+
+ args = ap.parse_args()
+ return args.func(args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())