diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:15:13 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:15:13 +0200 |
| commit | a8a89b3ed3101981d4d87ce50f924ea499c369fd (patch) | |
| tree | f3a3b51fa2eb6612652dfbc26bd9ee8527be2822 /bench | |
| parent | ed979a49940b3054468b48ad1a9f97fab5a2fd81 (diff) | |
| download | ele-a8a89b3ed3101981d4d87ce50f924ea499c369fd.tar.gz ele-a8a89b3ed3101981d4d87ce50f924ea499c369fd.zip | |
Optimize release throughput and add reproducible CLI benchmarks
Diffstat (limited to 'bench')
| -rw-r--r-- | bench/benchmark.py | 77 |
1 files changed, 77 insertions, 0 deletions
diff --git a/bench/benchmark.py b/bench/benchmark.py new file mode 100644 index 0000000..4fe208d --- /dev/null +++ b/bench/benchmark.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Compare CLI throughput and verify identical output on generated JSON lines.""" + +import argparse +import hashlib +import json +from pathlib import Path +import statistics +import subprocess +import time + +ROOT = Path(__file__).resolve().parent.parent +CASES = { + "bare": "needle", + "equality": "state = open", + "numeric": "priority > 50 AND score < 3.5", + "nested": "user.settings.theme = dark", + "list": "tags:beta", + "regex": 'msg =~ "[a-z]+ needle[0-9]+$"', + "regex_group": 'msg =~ ("[a-z]+ needle[0-9]+$" OR "^skip")', +} + + +def digest(binary, expression, data): + result = subprocess.run( + [str(binary), expression, str(data)], capture_output=True, check=True + ) + return hashlib.sha256(result.stdout).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path) + parser.add_argument("candidate", type=Path) + parser.add_argument("--records", type=int, default=100_000) + parser.add_argument("--runs", type=int, default=5) + args = parser.parse_args() + if args.records <= 0 or args.runs <= 0: + parser.error("records and runs must be positive") + binaries = [args.baseline.resolve(), args.candidate.resolve()] + data = ROOT / "target" / "benchmarks" / "records.jsonl" + data.parent.mkdir(parents=True, exist_ok=True) + with data.open("w") as output: + for i in range(args.records): + record = { + "id": i, + "msg": f"hello needle{i}" if i % 5 == 0 else "skip this record", + "state": "open" if i % 2 else "closed", + "priority": i % 100, + "score": (i % 10) / 2, + "user": {"settings": {"theme": "dark" if i % 3 else "light"}}, + "tags": ["stable", "beta" if i % 4 else "alpha"], + } + output.write(json.dumps(record, separators=(",", ":")) + "\n") + print(f"{args.records:,} records; {data.stat().st_size:,} bytes; median of {args.runs} runs") + print("case baseline(s) candidate(s) speedup") + for label, expression in CASES.items(): + assert digest(binaries[0], expression, data) == digest(binaries[1], expression, data), label + samples = [[], []] + for run in range(args.runs): + for index in ([0, 1] if run % 2 == 0 else [1, 0]): + start = time.perf_counter() + subprocess.run( + [str(binaries[index]), expression, str(data)], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + check=True, + ) + samples[index].append(time.perf_counter() - start) + before, after = (statistics.median(sample) for sample in samples) + print(f"{label:13} {before:11.4f} {after:12.4f} {before / after:6.2f}x") + for binary in binaries: + print(f"{binary.name}: {binary.stat().st_size:,} bytes") + + +if __name__ == "__main__": + main() |
