Bruno Postle
2ef2b15fe3
oracle.score_batch() writes/cleans N outputs and runs urb-fitness.pl once with all file names; oracle.score() is now a thin wrapper. Adds Score.fail_lines (sorted) because Perl hash-order randomisation shuffles .fails line order between runs, and documents Urb's ~1-ULP score nondeterminism (compare with rel tolerance, never ==). experiments/bench_batch_oracle.py validates batch-vs-single parity on the 35-file corpus and benchmarks: 0.98 s/dom batched vs 1.27 s/dom single (x1.30), all files identical (fitness to 1e-12 rel, exact failure sets). Closes homemaker-py-av5.
67 lines
2.6 KiB
Python
67 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate and benchmark the batched oracle (homemaker-py-av5).
|
|
|
|
Copies the 35-file programme-house corpus into a scratch directory, scores
|
|
every file via single-file oracle calls and again via one batched invocation,
|
|
then checks the per-file fitness and failure sets are identical and reports
|
|
measured s/dom for both modes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from homemaker import oracle # noqa: E402
|
|
|
|
URB = Path("/home/bruno/src/urb")
|
|
CORPUS = URB / "examples" / "programme-house"
|
|
|
|
|
|
def main() -> int:
|
|
with tempfile.TemporaryDirectory(prefix="bench_batch_") as tmp:
|
|
scratch = Path(tmp)
|
|
shutil.copy(CORPUS / "patterns.config", scratch)
|
|
doms = sorted(CORPUS.glob("*.dom"))
|
|
paths = [shutil.copy(d, scratch) for d in doms]
|
|
paths = [Path(p) for p in paths]
|
|
print(f"{len(paths)} corpus files -> {scratch}")
|
|
|
|
t0 = time.perf_counter()
|
|
singles = [oracle.score(p, URB) for p in paths]
|
|
t_single = time.perf_counter() - t0
|
|
|
|
t0 = time.perf_counter()
|
|
batch = oracle.score_batch(paths, URB)
|
|
t_batch = time.perf_counter() - t0
|
|
|
|
# Urb's score is nondeterministic at the ~1 ULP level (Perl hash-order
|
|
# summation), so compare fitness with a tight relative tolerance; any
|
|
# real semantic difference (e.g. occlusion handling) would be far larger.
|
|
mismatches = 0
|
|
for p, s, b in zip(paths, singles, batch):
|
|
if not math.isclose(s.fitness, b.fitness, rel_tol=1e-12) or s.fail_lines != b.fail_lines:
|
|
mismatches += 1
|
|
print(f"MISMATCH {p.name}: single {s.fitness:.12g} ({s.n_fails} fails) "
|
|
f"vs batch {b.fitness:.12g} ({b.n_fails} fails)")
|
|
print(f" single-only: {sorted(set(s.fail_lines) - set(b.fail_lines))}")
|
|
print(f" batch-only: {sorted(set(b.fail_lines) - set(s.fail_lines))}")
|
|
|
|
n = len(paths)
|
|
print(f"\nsingle-file: {t_single:.2f} s total, {t_single / n:.3f} s/dom")
|
|
print(f"batched: {t_batch:.2f} s total, {t_batch / n:.3f} s/dom")
|
|
print(f"speedup: x{t_single / t_batch:.2f}")
|
|
if mismatches:
|
|
print(f"\nFAIL: {mismatches}/{n} files differ between single and batch")
|
|
return 1
|
|
print(f"\nOK: all {n} files identical (fitness to 1e-12 rel, exact failure set) in both modes")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|