Bruno Postle
646ee30ab6
- src/homemaker/ → src/homemaker_layout/; all imports updated - pyproject.toml: name = homemaker-layout, entry point updated - .beads/config.yaml: dolt sync.remote updated to homemaker-layout.git - Delete temporary debug/perl scripts from project root - README.md, DESIGN.md: package path references updated - GitHub repo renamed; git remote updated Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
224 lines
8.4 KiB
Python
224 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
||
"""Inner-loop optimiser bake-off at equal oracle budgets (homemaker-py-d0s).
|
||
|
||
DESIGN.md §7 Phase 1(b), §8.3. DOF is only ≈ rooms−1 (6–7 on the corpus), so
|
||
the question is fitness gained per oracle evaluation, not asymptotic power.
|
||
Candidates:
|
||
|
||
nm multi-start Nelder-Mead (scipy) — the §4.5 diagnostic optimiser.
|
||
Inherently sequential: ONE dom per oracle call, so the Perl
|
||
startup never amortises (§4.6).
|
||
cma multi-phase CMA-ES (innerloop.cma_search), one batched oracle
|
||
call per generation.
|
||
compass single-start batched compass search with pattern moves + random
|
||
augmentation (innerloop.compass_search).
|
||
compass-ms multi-start compass: budget split across restarts (x0 first,
|
||
then random starts), global best kept.
|
||
|
||
Protocol: cold start from each file's equal-offset projection (x_current),
|
||
one run per (method, file, seed), best-so-far recorded after every oracle
|
||
call. Fitness-at-budget-B is read off the trace (evals ≤ B), so methods are
|
||
compared at exactly equal budgets regardless of batch granularity; checkpoint
|
||
budgets bracket the driver's real operating points (child_budget=80 warm,
|
||
seed_budget=200 cold). Wall-clock and oracle-invocation counts are recorded
|
||
per run.
|
||
|
||
Runs under URB_NO_OCCLUSION=1 (set by this script — the gp2 re-baseline flag;
|
||
all benchmarks must use it).
|
||
|
||
Usage: python3 experiments/bakeoff_innerloop.py [budget] [out.json]
|
||
(defaults: budget 200, experiments/bakeoff_innerloop.json)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import shutil
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||
from homemaker_layout import dom, innerloop, oracle # noqa: E402
|
||
|
||
URB = Path("/home/bruno/src/urb")
|
||
EX = URB / "examples" / "programme-house"
|
||
|
||
FILES = (
|
||
"2f45907abd9accac2a124d311732f749.dom",
|
||
"candidate-002.dom",
|
||
"c964435454c459f86c3ed9a5a7621132.dom",
|
||
)
|
||
SEEDS = (0, 1, 2)
|
||
CHECKPOINTS = (40, 80, 120, 200)
|
||
|
||
|
||
class TracingEvaluator(innerloop.OracleEvaluator):
|
||
"""Records (cumulative evals, batch-best fitness) after every oracle call."""
|
||
|
||
def __init__(self, *a, **kw):
|
||
super().__init__(*a, **kw)
|
||
self.trace: list[tuple[int, float]] = []
|
||
|
||
def evaluate(self, xs):
|
||
scores = super().evaluate(xs)
|
||
self.trace.append((self.n_evals, max(s.fitness for s in scores)))
|
||
return scores
|
||
|
||
def best_at(self, budget: int) -> float:
|
||
vals = [f for n, f in self.trace if n <= budget]
|
||
return max(vals) if vals else float("nan")
|
||
|
||
|
||
class _BudgetExhausted(Exception):
|
||
pass
|
||
|
||
|
||
def nm_search(ev, x0, budget=200, seed=0):
|
||
"""Multi-start Nelder-Mead: x0 first, random restarts until the budget is
|
||
spent. Sequential — every evaluation is its own oracle invocation."""
|
||
from scipy.optimize import minimize
|
||
|
||
rng = np.random.default_rng(seed)
|
||
n = len(x0)
|
||
x = np.clip(np.asarray(x0, dtype=float), innerloop._EPS, 1 - innerloop._EPS)
|
||
s = ev.evaluate([x])[0]
|
||
best = innerloop.Result(
|
||
x=x.copy(), fitness=s.fitness, n_fails=s.n_fails, fail_lines=s.fail_lines,
|
||
x0_fitness=s.fitness, x0_n_fails=s.n_fails, n_evals=0, n_oracle_calls=0,
|
||
)
|
||
|
||
def f(xi):
|
||
if ev.n_evals >= budget:
|
||
raise _BudgetExhausted
|
||
sc = ev.evaluate([np.asarray(xi, dtype=float)])[0]
|
||
if sc.fitness > best.fitness:
|
||
best.x = np.asarray(xi, dtype=float).copy()
|
||
best.fitness = sc.fitness
|
||
best.n_fails = sc.n_fails
|
||
best.fail_lines = sc.fail_lines
|
||
return -sc.fitness
|
||
|
||
start = x.copy()
|
||
while ev.n_evals < budget:
|
||
try:
|
||
minimize(
|
||
f, start, method="Nelder-Mead",
|
||
bounds=[(innerloop._EPS, 1 - innerloop._EPS)] * n,
|
||
options={"maxfev": budget - ev.n_evals, "xatol": 1e-3, "fatol": 1e-10},
|
||
)
|
||
except _BudgetExhausted:
|
||
break
|
||
start = rng.uniform(0.1, 0.9, n) # restart
|
||
|
||
best.n_evals = ev.n_evals
|
||
best.n_oracle_calls = ev.n_oracle_calls
|
||
return best
|
||
|
||
|
||
def compass_ms_search(ev, x0, budget=200, seed=0, n_starts=3):
|
||
"""Multi-start compass: budget split evenly; first start is x0, the rest
|
||
random. compass_search counts against ev.n_evals, so phase budgets are
|
||
cumulative caps."""
|
||
rng = np.random.default_rng(seed)
|
||
n = len(x0)
|
||
best = None
|
||
for phase in range(n_starts):
|
||
phase_end = ev.n_evals + (budget - ev.n_evals) // (n_starts - phase)
|
||
start = np.asarray(x0, dtype=float) if phase == 0 else rng.uniform(0.1, 0.9, n)
|
||
r = innerloop.compass_search(ev, start, budget=phase_end, seed=seed + phase)
|
||
if best is None or r.fitness > best.fitness:
|
||
keep_x0 = best.x0_fitness if best is not None else r.x0_fitness
|
||
keep_x0f = best.x0_n_fails if best is not None else r.x0_n_fails
|
||
best = r
|
||
best.x0_fitness, best.x0_n_fails = keep_x0, keep_x0f
|
||
if ev.n_evals >= budget:
|
||
break
|
||
best.n_evals = ev.n_evals
|
||
best.n_oracle_calls = ev.n_oracle_calls
|
||
return best
|
||
|
||
|
||
METHODS = {
|
||
"nm": nm_search,
|
||
"cma": innerloop.cma_search,
|
||
"compass": innerloop.compass_search,
|
||
"compass-ms": compass_ms_search,
|
||
}
|
||
|
||
|
||
def main() -> int:
|
||
budget = int(sys.argv[1]) if len(sys.argv) > 1 else 200
|
||
out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else (
|
||
Path(__file__).parent / "bakeoff_innerloop.json")
|
||
os.environ["URB_NO_OCCLUSION"] = "1"
|
||
checkpoints = [c for c in CHECKPOINTS if c <= budget] or [budget]
|
||
if checkpoints[-1] != budget:
|
||
checkpoints.append(budget)
|
||
|
||
# Baselines: the UNMODIFIED originals (gains measured from there, not from
|
||
# the equal-offset projection — accept_innerloop.py convention).
|
||
orig: dict[str, oracle.Score] = {}
|
||
with tempfile.TemporaryDirectory(prefix="bakeoff_orig_") as tmp:
|
||
shutil.copy(EX / "patterns.config", tmp)
|
||
for name in FILES:
|
||
orig[name] = oracle.score(shutil.copy(EX / name, tmp), URB)
|
||
|
||
runs = []
|
||
for name in FILES:
|
||
for method in METHODS:
|
||
for seed in SEEDS:
|
||
root = dom.load(str(EX / name))
|
||
with TracingEvaluator(root, EX, URB) as ev:
|
||
x0 = ev.x_current
|
||
t0 = time.perf_counter()
|
||
r = METHODS[method](ev, x0, budget=budget, seed=seed)
|
||
dt = time.perf_counter() - t0
|
||
run = {
|
||
"file": name, "method": method, "seed": seed,
|
||
"dof": len(x0),
|
||
"orig_fitness": orig[name].fitness,
|
||
"orig_n_fails": orig[name].n_fails,
|
||
"x0_fitness": r.x0_fitness, "x0_n_fails": r.x0_n_fails,
|
||
"best_at": {str(c): ev.best_at(c) for c in checkpoints},
|
||
"final_fitness": r.fitness, "final_n_fails": r.n_fails,
|
||
"n_evals": ev.n_evals, "n_oracle_calls": ev.n_oracle_calls,
|
||
"wall_s": dt,
|
||
}
|
||
runs.append(run)
|
||
gains = " ".join(
|
||
f"@{c}:x{run['best_at'][str(c)] / orig[name].fitness:.2f}"
|
||
for c in checkpoints)
|
||
print(
|
||
f"{name[:12]:12s} {method:10s} seed={seed} {gains} "
|
||
f"fails {orig[name].n_fails}->{r.n_fails} "
|
||
f"{ev.n_evals}ev/{ev.n_oracle_calls}calls {dt:.0f}s",
|
||
flush=True,
|
||
)
|
||
|
||
out_path.write_text(json.dumps(
|
||
{"budget": budget, "checkpoints": checkpoints, "runs": runs}, indent=1))
|
||
print(f"\nwrote {out_path}")
|
||
|
||
# Summary: mean gain over original at each checkpoint, mean s/eval.
|
||
print(f"\n{'method':10s} " + "".join(f"{'x@' + str(c):>8s}" for c in checkpoints)
|
||
+ f"{'s/eval':>8s}{'calls':>7s}{'fails+':>7s}")
|
||
for method in METHODS:
|
||
rs = [r for r in runs if r["method"] == method]
|
||
cols = ""
|
||
for c in checkpoints:
|
||
g = np.mean([r["best_at"][str(c)] / r["orig_fitness"] for r in rs])
|
||
cols += f"{g:8.2f}"
|
||
spe = np.mean([r["wall_s"] / r["n_evals"] for r in rs])
|
||
calls = np.mean([r["n_oracle_calls"] for r in rs])
|
||
newf = sum(r["final_n_fails"] > r["orig_n_fails"] for r in rs)
|
||
print(f"{method:10s} {cols}{spe:8.2f}{calls:7.0f}{newf:7d}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|