179 lines
6.6 KiB
Python
179 lines
6.6 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""Penalty-reshaping validation: lexicographic outer search (homemaker-py-yg5).
|
|||
|
|
|
|||
|
|
DESIGN.md §4.8, §5.4, §7 Phase 4, §8.5.
|
|||
|
|
|
|||
|
|
Two measurements:
|
|||
|
|
|
|||
|
|
1. INNER-LOOP PROTECTION: run nm_search at budget 80 on corpus files × seeds.
|
|||
|
|
Asserts x0_n_fails >= result.n_fails for every run (cliff NEVER allows a new
|
|||
|
|
failure to enter). The inner loop code is unchanged; this confirms the 0.5^n
|
|||
|
|
cliff still guards it.
|
|||
|
|
|
|||
|
|
2. OUTER-SEARCH QUALITY: run driver.search at a modest budget with lex=True
|
|||
|
|
(lexicographic: fewer fails first, then score) vs lex=False (old scalar
|
|||
|
|
fitness comparison) across several rng seeds. Reports: mean evals to first
|
|||
|
|
improvement, mean best score, mean best n_fails at budget.
|
|||
|
|
|
|||
|
|
Runs under URB_NO_OCCLUSION=1.
|
|||
|
|
|
|||
|
|
Usage:
|
|||
|
|
python3 experiments/penalty_reshape.py [outer_budget] [out.json]
|
|||
|
|
(defaults: outer_budget 2000, experiments/penalty_reshape.json)
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import sys
|
|||
|
|
import time
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
import numpy as np
|
|||
|
|
|
|||
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|||
|
|
from homemaker_layout import dom, driver, innerloop # noqa: E402
|
|||
|
|
|
|||
|
|
os.environ.setdefault("URB_NO_OCCLUSION", "1")
|
|||
|
|
|
|||
|
|
EX = Path("/home/bruno/src/urb/examples/programme-house")
|
|||
|
|
|
|||
|
|
# Files used for inner-loop protection test (same as bakeoff_native.py baseline)
|
|||
|
|
IL_FILES = (
|
|||
|
|
"2f45907abd9accac2a124d311732f749.dom",
|
|||
|
|
"candidate-002.dom",
|
|||
|
|
"c964435454c459f86c3ed9a5a7621132.dom",
|
|||
|
|
)
|
|||
|
|
IL_SEEDS = (0, 1, 2)
|
|||
|
|
IL_BUDGET = 80
|
|||
|
|
|
|||
|
|
# Seed file for outer-search comparison — 2f45907 starts with 2 fails after
|
|||
|
|
# inner loop (inner loop achieves 2 fails at budget 80 on this file), so the
|
|||
|
|
# initial population mixes 2-fail and 3-fail designs and the cross-tier
|
|||
|
|
# comparison between lex and scalar is exercised.
|
|||
|
|
OUTER_SEED_FILE = EX / "2f45907abd9accac2a124d311732f749.dom"
|
|||
|
|
OUTER_SEEDS = (0, 1, 2)
|
|||
|
|
OUTER_POP = 8
|
|||
|
|
OUTER_CHILD_BUDGET = 80
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# Part 1 — inner-loop protection
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def run_inner_loop_protection(budget: int = IL_BUDGET) -> list[dict]:
|
|||
|
|
print("\n=== Part 1: inner-loop protection ===")
|
|||
|
|
print(f"{'file':24s} {'seed':>4s} {'x0_f':>6s} {'fin_f':>6s} {'x0_fit':>10s} "
|
|||
|
|
f"{'fin_fit':>10s} {'ok':>4s}")
|
|||
|
|
results = []
|
|||
|
|
for fname in IL_FILES:
|
|||
|
|
root_orig = dom.load(str(EX / fname))
|
|||
|
|
for seed in IL_SEEDS:
|
|||
|
|
root = dom.load(str(EX / fname))
|
|||
|
|
with innerloop.NativeEvaluator(root, EX) as ev:
|
|||
|
|
x0 = ev.x_current
|
|||
|
|
r = innerloop.nm_search(ev, x0, budget=budget, seed=seed)
|
|||
|
|
ok = r.n_fails <= r.x0_n_fails
|
|||
|
|
print(f"{fname[:24]:24s} {seed:4d} {r.x0_n_fails:6d} {r.n_fails:6d} "
|
|||
|
|
f"{r.x0_fitness:10.5f} {r.fitness:10.5f} {'OK' if ok else 'FAIL':>4s}")
|
|||
|
|
results.append({
|
|||
|
|
"file": fname,
|
|||
|
|
"seed": seed,
|
|||
|
|
"x0_n_fails": r.x0_n_fails,
|
|||
|
|
"final_n_fails": r.n_fails,
|
|||
|
|
"x0_fitness": r.x0_fitness,
|
|||
|
|
"final_fitness": r.fitness,
|
|||
|
|
"regression": not ok,
|
|||
|
|
})
|
|||
|
|
n_reg = sum(r["regression"] for r in results)
|
|||
|
|
print(f"\nFail regressions: {n_reg}/{len(results)} "
|
|||
|
|
f"({'PASS — inner loop protected' if n_reg == 0 else 'FAIL — cliff broken'})")
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# Part 2 — outer-search quality comparison
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def run_outer_search(budget: int, use_lex: bool, seed: int) -> dict:
|
|||
|
|
root = dom.load(str(OUTER_SEED_FILE))
|
|||
|
|
t0 = time.perf_counter()
|
|||
|
|
r = driver.search(
|
|||
|
|
root,
|
|||
|
|
EX,
|
|||
|
|
budget=budget,
|
|||
|
|
pop_size=OUTER_POP,
|
|||
|
|
child_budget=OUTER_CHILD_BUDGET,
|
|||
|
|
seed=seed,
|
|||
|
|
use_lex=use_lex,
|
|||
|
|
)
|
|||
|
|
wall = time.perf_counter() - t0
|
|||
|
|
first_improvement_evals = r.history[1][0] if len(r.history) > 1 else None
|
|||
|
|
return {
|
|||
|
|
"scheme": "lex" if use_lex else "scalar",
|
|||
|
|
"seed": seed,
|
|||
|
|
"budget": budget,
|
|||
|
|
"best_fitness": r.best.fitness if r.best else None,
|
|||
|
|
"best_n_fails": r.best.n_fails if r.best else None,
|
|||
|
|
"n_evals": r.n_evals,
|
|||
|
|
"n_topologies": r.n_topologies,
|
|||
|
|
"n_improvements": len(r.history),
|
|||
|
|
"first_improvement_evals": first_improvement_evals,
|
|||
|
|
"wall_s": wall,
|
|||
|
|
"history": [(ev, fit, lin) for ev, fit, lin in r.history],
|
|||
|
|
"population": [(p.fitness, p.n_fails) for p in r.population],
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def run_outer_comparison(budget: int) -> list[dict]:
|
|||
|
|
print(f"\n=== Part 2: outer-search comparison (budget {budget}) ===")
|
|||
|
|
print(f"{'scheme':8s} {'seed':>4s} {'best_fit':>12s} {'n_fails':>7s} "
|
|||
|
|
f"{'topologies':>10s} {'improvements':>12s} {'1st_improv':>10s}")
|
|||
|
|
results = []
|
|||
|
|
for use_lex in (True, False):
|
|||
|
|
for seed in OUTER_SEEDS:
|
|||
|
|
r = run_outer_search(budget, use_lex, seed)
|
|||
|
|
results.append(r)
|
|||
|
|
first = r["first_improvement_evals"]
|
|||
|
|
print(f"{'lex' if use_lex else 'scalar':8s} {seed:4d} "
|
|||
|
|
f"{r['best_fitness']:12.6g} {r['best_n_fails']:7d} "
|
|||
|
|
f"{r['n_topologies']:10d} {r['n_improvements']:12d} "
|
|||
|
|
f"{str(first):>10s}")
|
|||
|
|
print()
|
|||
|
|
for scheme in ("lex", "scalar"):
|
|||
|
|
rs = [r for r in results if r["scheme"] == scheme]
|
|||
|
|
mean_fit = np.mean([r["best_fitness"] for r in rs])
|
|||
|
|
mean_fails = np.mean([r["best_n_fails"] for r in rs])
|
|||
|
|
mean_topo = np.mean([r["n_topologies"] for r in rs])
|
|||
|
|
first_evs = [r["first_improvement_evals"] for r in rs if r["first_improvement_evals"]]
|
|||
|
|
mean_first = np.mean(first_evs) if first_evs else float("nan")
|
|||
|
|
print(f"{scheme:8s} mean_fit={mean_fit:.5g} mean_fails={mean_fails:.2f} "
|
|||
|
|
f"mean_topologies={mean_topo:.0f} mean_first_improvement={mean_first:.0f}")
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
# Main
|
|||
|
|
# ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
def main() -> int:
|
|||
|
|
outer_budget = int(sys.argv[1]) if len(sys.argv) > 1 else 2000
|
|||
|
|
out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else (
|
|||
|
|
Path(__file__).parent / "penalty_reshape.json")
|
|||
|
|
|
|||
|
|
il_results = run_inner_loop_protection()
|
|||
|
|
outer_results = run_outer_comparison(outer_budget)
|
|||
|
|
|
|||
|
|
out = {
|
|||
|
|
"inner_loop_protection": il_results,
|
|||
|
|
"outer_search": outer_results,
|
|||
|
|
}
|
|||
|
|
out_path.write_text(json.dumps(out, indent=1))
|
|||
|
|
print(f"\nwrote {out_path}")
|
|||
|
|
return 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
sys.exit(main())
|