homemaker-layout/experiments/bakeoff_harbor.py

#!/usr/bin/env python3
"""Inner-loop bake-off on harbor-house designs (homemaker-py-d6d).

Extends bakeoff_native.py to harbor-house, which has 3–40 DOF vs programme-
house's 6–7.  Tests whether NM's early-budget advantage holds as DOF grows.
Files are grouped by DOF to show the scaling behaviour clearly.

Usage: python3 experiments/bakeoff_harbor.py [budget] [out.json]
       (defaults: budget 200, experiments/bakeoff_harbor.json)
"""

from __future__ import annotations

import json
import os
import sys
import time
from pathlib import Path

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from homemaker_layout import dom, fitness as fit_mod, innerloop, solver  # noqa: E402

EX = Path("/home/bruno/src/urb/examples/harbor-house")

# All files with DOF >= 3, sorted by DOF ascending
FILES = (
    "43a539866a2b63ff77c8fa11f92e133c.dom",   #  3 DOF
    "59a65b704b461146e8e2efaec9013e39.dom",   #  5 DOF
    "ec1f082320cbdbb2c5b24e29dbd4e0d0.dom",   # 11 DOF
    "448d535590f29d65a0469fb4ecbf4b56.dom",   # 14 DOF
    "0f931851fa0fd5fec70db5ae2899f10a.dom",   # 23 DOF
    "c07a3c3ccaccf580227fb6acfef8b263.dom",   # 35 DOF
    "dfa595104a9ac8a903db309697679455.dom",   # 39 DOF
    "2b51b0402ee38c150716673894d8f5c0.dom",   # 40 DOF
    "71d93882d8a520dc6c4e6fa1bcaea33a.dom",   # 40 DOF
)
SEEDS = (0, 1, 2)
CHECKPOINTS = (40, 80, 120, 200)


class NativeTracingEvaluator(innerloop.NativeEvaluator):
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        self.trace: list[tuple[int, float]] = []

    def evaluate(self, xs):
        scores = super().evaluate(xs)
        self.trace.append((self.n_evals, max(s.fitness for s in scores)))
        return scores

    def best_at(self, budget: int) -> float:
        vals = [f for n, f in self.trace if n <= budget]
        return max(vals) if vals else float("nan")


class _BudgetExhausted(Exception):
    pass


def nm_search(ev, x0, budget=200, seed=0):
    from scipy.optimize import minimize
    rng = np.random.default_rng(seed)
    n = len(x0)
    x = np.clip(np.asarray(x0, dtype=float), innerloop._EPS, 1 - innerloop._EPS)
    s = ev.evaluate([x])[0]
    best = innerloop.Result(
        x=x.copy(), fitness=s.fitness, n_fails=s.n_fails, fail_lines=s.fail_lines,
        x0_fitness=s.fitness, x0_n_fails=s.n_fails, n_evals=0, n_oracle_calls=0,
    )

    def f(xi):
        if ev.n_evals >= budget:
            raise _BudgetExhausted
        sc = ev.evaluate([np.asarray(xi, dtype=float)])[0]
        if sc.fitness > best.fitness:
            best.x = np.asarray(xi, dtype=float).copy()
            best.fitness = sc.fitness
            best.n_fails = sc.n_fails
            best.fail_lines = sc.fail_lines
        return -sc.fitness

    start = x.copy()
    while ev.n_evals < budget:
        try:
            minimize(
                f, start, method="Nelder-Mead",
                bounds=[(innerloop._EPS, 1 - innerloop._EPS)] * n,
                options={"maxfev": budget - ev.n_evals, "xatol": 1e-3, "fatol": 1e-10},
            )
        except _BudgetExhausted:
            break
        start = rng.uniform(0.1, 0.9, n)

    best.n_evals = ev.n_evals
    best.n_oracle_calls = ev.n_oracle_calls
    return best


def compass_ms_search(ev, x0, budget=200, seed=0, n_starts=3):
    rng = np.random.default_rng(seed)
    n = len(x0)
    best = None
    for phase in range(n_starts):
        phase_end = ev.n_evals + (budget - ev.n_evals) // (n_starts - phase)
        start = np.asarray(x0, dtype=float) if phase == 0 else rng.uniform(0.1, 0.9, n)
        r = innerloop.compass_search(ev, start, budget=phase_end, seed=seed + phase)
        if best is None or r.fitness > best.fitness:
            keep_x0 = best.x0_fitness if best is not None else r.x0_fitness
            keep_x0f = best.x0_n_fails if best is not None else r.x0_n_fails
            best = r
            best.x0_fitness, best.x0_n_fails = keep_x0, keep_x0f
        if ev.n_evals >= budget:
            break
    best.n_evals = ev.n_evals
    best.n_oracle_calls = ev.n_oracle_calls
    return best


METHODS = {
    "nm": nm_search,
    "cma": innerloop.cma_search,
    "compass": innerloop.compass_search,
    "compass-ms": compass_ms_search,
}


def native_baseline(path: Path) -> innerloop._NativeScore:
    root = dom.load(str(path))
    conf, cost = fit_mod.load_config(EX)
    fit = fit_mod.Fitness(conf, cost)
    score, fails = fit.score_with_fails(root)
    return innerloop._NativeScore(fitness=score, fail_lines=tuple(fails))


def dof_of(path: Path) -> int:
    root = dom.load(str(path))
    return len(solver.free_branches(root))


def main() -> int:
    budget = int(sys.argv[1]) if len(sys.argv) > 1 else 200
    out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else (
        Path(__file__).parent / "bakeoff_harbor.json")
    os.environ["URB_NO_OCCLUSION"] = "1"
    checkpoints = [c for c in CHECKPOINTS if c <= budget] or [budget]
    if checkpoints[-1] != budget:
        checkpoints.append(budget)

    file_dof = {name: dof_of(EX / name) for name in FILES}
    orig = {name: native_baseline(EX / name) for name in FILES}

    runs = []
    for name in FILES:
        dof = file_dof[name]
        for method in METHODS:
            for seed in SEEDS:
                root = dom.load(str(EX / name))
                ev = NativeTracingEvaluator(root, EX)
                x0 = ev.x_current
                t0 = time.perf_counter()
                r = METHODS[method](ev, x0, budget=budget, seed=seed)
                dt = time.perf_counter() - t0
                run = {
                    "file": name, "method": method, "seed": seed,
                    "dof": dof,
                    "orig_fitness": orig[name].fitness,
                    "orig_n_fails": orig[name].n_fails,
                    "x0_fitness": r.x0_fitness, "x0_n_fails": r.x0_n_fails,
                    "best_at": {str(c): ev.best_at(c) for c in checkpoints},
                    "final_fitness": r.fitness, "final_n_fails": r.n_fails,
                    "n_evals": ev.n_evals,
                    "wall_s": dt,
                }
                runs.append(run)
                gains = "  ".join(
                    f"@{c}:x{run['best_at'][str(c)] / orig[name].fitness:.2f}"
                    for c in checkpoints)
                print(
                    f"DOF={dof:2d}  {method:10s}  seed={seed}  {gains}  "
                    f"fails {orig[name].n_fails}->{r.n_fails}  {ev.n_evals}ev  {dt:.1f}s",
                    flush=True,
                )

    out_path.write_text(json.dumps(
        {"budget": budget, "checkpoints": checkpoints, "runs": runs}, indent=1))
    print(f"\nwrote {out_path}")

    # Summary by DOF band
    bands = [(1, 10, "small (≤10)"), (11, 20, "med (11-20)"), (21, 50, "large (>20)")]
    for lo, hi, label in bands:
        band_runs = [r for r in runs if lo <= r["dof"] <= hi]
        if not band_runs:
            continue
        print(f"\n--- {label} ---")
        print(f"{'method':10s} " + "".join(f"{'x@' + str(c):>8s}" for c in checkpoints)
              + f"  s/eval  fails+  n")
        for method in METHODS:
            rs = [r for r in band_runs if r["method"] == method]
            if not rs:
                continue
            cols = ""
            for c in checkpoints:
                g = np.mean([r["best_at"][str(c)] / r["orig_fitness"] for r in rs])
                cols += f"{g:8.3f}"
            spe = np.mean([r["wall_s"] / max(r["n_evals"], 1) for r in rs])
            newf = sum(r["final_n_fails"] > r["orig_n_fails"] for r in rs)
            print(f"{method:10s} {cols}  {spe:.4f}  {newf:5d}  {len(rs)}")

    return 0


if __name__ == "__main__":
    sys.exit(main())