diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:03:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:13:28 +0200 |
| commit | 64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch) | |
| tree | 85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /tools/baseline.py | |
| download | pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip | |
Init
Diffstat (limited to 'tools/baseline.py')
| -rw-r--r-- | tools/baseline.py | 137 |
1 files changed, 137 insertions, 0 deletions
diff --git a/tools/baseline.py b/tools/baseline.py new file mode 100644 index 0000000..08ae72a --- /dev/null +++ b/tools/baseline.py @@ -0,0 +1,137 @@ +"""PyTorch baseline measurement and golden-vector capture. + +Usage: + python tools/baseline.py bench [--batch N] + python tools/baseline.py golden <audio.wav> <out.npz> [--chunks N] + +The benchmark protocol is fixed here and mirrored by bench/bench.c so the two +numbers are comparable: 3 warm-up forwards, then 20 timed forwards on the same +input, reporting mean, p50 and p99 in milliseconds plus the realtime factor. +""" + +import argparse +import resource +import statistics +import sys +import time + +import numpy as np + +from model import NUM_FRAMES, NUM_SAMPLES, SAMPLE_RATE, checkpoint_path + + +def load_torch_model(): + import torch + from pyannote.audio import Model + + torch.set_num_threads(1) + model = Model.from_pretrained(checkpoint_path()) + model.eval() + return model + + +def read_wav(path: str) -> np.ndarray: + import soundfile as sf + + data, rate = sf.read(path, dtype="float32", always_2d=True) + if rate != SAMPLE_RATE: + raise SystemExit(f"{path}: expected {SAMPLE_RATE} Hz, got {rate}") + return np.ascontiguousarray(data[:, 0]) + + +def bench(args) -> int: + import torch + + t0 = time.perf_counter() + model = load_torch_model() + init_ms = (time.perf_counter() - t0) * 1e3 + + rng = np.random.default_rng(1) + wav = rng.standard_normal((args.batch, 1, NUM_SAMPLES)).astype(np.float32) * 0.1 + x = torch.from_numpy(wav) + + with torch.no_grad(): + for _ in range(3): + model(x) + times = [] + for _ in range(20): + t = time.perf_counter() + model(x) + times.append((time.perf_counter() - t) * 1e3) + + times.sort() + mean = statistics.fmean(times) + p50 = times[len(times) // 2] + p99 = times[min(len(times) - 1, int(0.99 * len(times)))] + audio_s = 10.0 * args.batch + rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e6 + + print(f"torch batch={args.batch} threads=1") + print(f" init {init_ms:8.1f} ms") + print(f" mean/p50/p99 {mean:8.2f} / {p50:.2f} / {p99:.2f} ms") + print(f" per chunk {mean / args.batch:8.2f} ms") + print(f" realtime factor {audio_s / (mean / 1e3):8.1f}x") + print(f" max rss {rss:8.1f} MB") + return 0 + + +def golden(args) -> int: + import torch + + from reference import Reference + from model import load_state_dict + + wav = read_wav(args.audio) + step = int(1.0 * SAMPLE_RATE) + starts = list(range(0, len(wav) - NUM_SAMPLES + 1, step))[: args.chunks] + if not starts: + raise SystemExit("audio shorter than one 10 s window") + + model = load_torch_model() + inputs = np.stack([wav[s : s + NUM_SAMPLES] for s in starts]) + + outs = [] + with torch.no_grad(): + for i in range(0, len(inputs), 8): + batch = torch.from_numpy(inputs[i : i + 8])[:, None, :] + outs.append(model(batch).numpy()) + outputs = np.concatenate(outs) + assert outputs.shape == (len(starts), NUM_FRAMES, 7) + + # A numpy-reference pass over the same inputs pins the tolerance the C + # engine is later judged against. + ref = Reference(load_state_dict()) + ref_out = np.stack([ref.forward(x) for x in inputs[: min(8, len(inputs))]]) + diff = np.abs(ref_out - outputs[: len(ref_out)]).max() + print(f"numpy reference vs torch on {len(ref_out)} chunks: max diff {diff:.3e}") + + np.savez_compressed( + args.out, + inputs=inputs.astype(np.float32), + outputs=outputs.astype(np.float32), + starts=np.array(starts, dtype=np.int64), + ) + print(f"wrote {args.out}: {len(starts)} chunks from {args.audio}") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + b = sub.add_parser("bench") + b.add_argument("--batch", type=int, default=1) + b.set_defaults(func=bench) + + g = sub.add_parser("golden") + g.add_argument("audio") + g.add_argument("out") + g.add_argument("--chunks", type=int, default=32) + g.set_defaults(func=golden) + + args = ap.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) |
