From d0567d7a7472f6cab07d066f5864b718dd7bcf7d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 07:56:24 +0000 Subject: [PATCH] Remove the Perl oracle Owner's decision: "we need to abandon the perl oracle, this was only useful when initially porting, but I suspect many of the remaining problems have been carried in from the perl (such as the weird scoring of outdoor and circulation space, which definitely needs fixing)". 39 supports that second clause. Every defect the section found is inherited, not introduced: the two-sided crinkliness gaussian that double-charges surplus daylight (39.14), quality as a product over a variable number of factors (39.18), value_supported priced as value_inside so a terrace was worth more per m2 than a room (39.19), and circulation returning 0.07 per unit cost (hxi). So parity with the oracle was never a safety net -- it was a commitment to reproduce those defects. Each of 39.14, 39.18 and 39.19 would have been a parity failure had parity ever been checked, and keeping the tests would have meant reverting the fixes or explaining the failures away. Removed: oracle.py, test_oracle.py, the two parity tests and their fixture machinery in test_dom_corpus.py, innerloop.OracleEvaluator with its use_native and urb_root plumbing, the same plumbing through driver, and fourteen experiments/ scripts that could only run against Perl. Several of those are cited in earlier DESIGN sections; the citations now point into git history, which is the honest state -- they had been unrunnable since the oracle root (/home/bruno/src/urb) stopped being present. run_search is superseded by run_search_scaled, which does the same job natively. Kept: dump_areas.pl/.py, which validate GEOMETRY against Urb (4.1) rather than fitness, and the prose in fitness_cmd.py and dom.py explaining why the .score/.fails formats are shaped as they are. Provenance is worth keeping; a dead code path is not. CLAUDE.md updated: fitness.py is the only evaluator, and "Urb did it this way" is no longer an argument that a constant is right. 39.16 is the standing counterweight in the other direction -- the crinkliness target WAS right and twice looked wrong only because the code reading it was misunderstood. Inheritance is neither evidence for nor against. 410 passed. The 69 removed cases account exactly: 64 parity (all skipped, since no oracle .score was ever committed), 4 in test_oracle.py, and the guard test 39.20 added as a stopgap. Closes homemaker-py-118. Files homemaker-py-bk9 for the re-baseline that 39.19 made necessary. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB --- CLAUDE.md | 14 +- DESIGN.md | 57 +++++++ experiments/accept_innerloop.py | 81 --------- experiments/bakeoff_innerloop.py | 224 ------------------------- experiments/bakeoff_native.py | 207 ----------------------- experiments/bench_batch_oracle.py | 67 -------- experiments/benchmark_vs_urbevolve.py | 133 --------------- experiments/diag_2f45907.py | 3 +- experiments/diag_depth_balance.py | 2 +- experiments/diag_leaf_sharing.py | 2 +- experiments/diag_slack_localization.py | 2 +- experiments/dump_leaf_quality.pl | 99 ----------- experiments/genome_parity.py | 56 ------- experiments/leaf_parity.py | 183 -------------------- experiments/operator_locality.py | 103 ------------ experiments/optimize_fullfitness.py | 89 ---------- experiments/rebaseline_no_occlusion.py | 71 -------- experiments/refine_sweep.py | 79 --------- experiments/resolve_ratios.py | 90 ---------- experiments/run_search.py | 61 ------- experiments/run_search_scaled.py | 1 - experiments/sweep_failtypes.py | 79 --------- src/homemaker_layout/driver.py | 17 +- src/homemaker_layout/fitness.py | 3 +- src/homemaker_layout/innerloop.py | 94 ++--------- src/homemaker_layout/oracle.py | 190 --------------------- tests/test_dom_corpus.py | 74 +------- tests/test_driver.py | 18 +- tests/test_innerloop.py | 21 ++- tests/test_oracle.py | 31 ---- 30 files changed, 126 insertions(+), 2025 deletions(-) delete mode 100644 experiments/accept_innerloop.py delete mode 100644 experiments/bakeoff_innerloop.py delete mode 100644 experiments/bakeoff_native.py delete mode 100644 experiments/bench_batch_oracle.py delete mode 100644 experiments/benchmark_vs_urbevolve.py delete mode 100644 experiments/dump_leaf_quality.pl delete mode 100644 experiments/genome_parity.py delete mode 100644 experiments/leaf_parity.py delete mode 100644 experiments/operator_locality.py delete mode 100644 experiments/optimize_fullfitness.py delete mode 100644 experiments/rebaseline_no_occlusion.py delete mode 100644 experiments/refine_sweep.py delete mode 100644 experiments/resolve_ratios.py delete mode 100644 experiments/run_search.py delete mode 100644 experiments/sweep_failtypes.py delete mode 100644 src/homemaker_layout/oracle.py delete mode 100644 tests/test_oracle.py diff --git a/CLAUDE.md b/CLAUDE.md index d464605..b18e605 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,7 +100,6 @@ Key modules: - `innerloop.py` — ratio optimisation inner loop (Nelder-Mead / CMA-ES) - `driver.py` — memetic search outer loop - `evolve.py` — `homemaker-evolve` CLI entry point -- `oracle.py` — legacy Perl shim, kept for validation only; do not use in new code - `bubble.py` — 3D bubble-diagram adjacency fitness-signal prototype (DESIGN.md §27, `mi7`); validated NULL, not wired into `fitness.py` — reference only, do not build on without a new formulation ## Conventions & Patterns @@ -119,5 +118,14 @@ homemaker-fitness cf0b8a77e8b2325f92a7e7d150184a55.dom The score is written to `.dom.score` and failures to `.dom.fails`; the numeric score is also printed to stderr. -Do **not** use `urb-fitness.pl` directly — `oracle.py` and the Perl tool are -kept only for cross-validation. +`fitness.py` is the **only** evaluator. The Perl oracle it was ported from — +`oracle.py`, `urb-fitness.pl`, and the parity tests against them — is gone +(DESIGN.md §39.21). Those parity tests had never actually run: no oracle +`.score` was ever committed, so on a clean checkout every case skipped, and the +only cases that ever executed compared the native scorer with itself (§39.20). + +The corollary matters when reading the objective: a constant or a rule that +looks odd is **not** thereby validated by "Urb did it this way". Several +defects found in §39 were carried straight over from the Perl — see §39.19 on +`value_supported`, and `homemaker-py-hxi` on circulation, which the owner has +ruled needs fixing. diff --git a/DESIGN.md b/DESIGN.md index 89cbd63..98514ac 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -7757,3 +7757,60 @@ misread constant and this all have the same shape: something was trusted because it was *present*, not because it was *checked*. The fix each time is to make the artefact carry its own justification — a committed fixture, a stated derivation, a naming convention that cannot admit the wrong file. + +### 39.21 The Perl oracle is gone (`homemaker-py-118`) + +§39.20 found that the native-vs-Perl parity tests had never run and, where they +did run, were comparing the native scorer with itself. The owner's decision on +that is to remove the oracle outright: *we need to abandon the perl oracle, +this was only useful when initially porting, but I suspect many of the +remaining problems have been carried in from the perl (such as the weird +scoring of outdoor and circulation space, which definitely needs fixing).* + +That second clause is the important one, and §39 supports it. Every defect this +section found is inherited, not introduced: + +| defect | where it came from | +|---|---| +| §39.14 two-sided crinkliness gaussian, surplus daylight double-charged | Urb | +| §39.18 quality as a product over a variable number of factors | Urb | +| §39.19 `value_supported` = `value_inside`, terrace priced as a room | Urb | +| `hxi` circulation returning 0.07 per unit cost | Urb | + +Parity with the oracle was therefore not a safety net; it was a **commitment to +reproduce those defects**. Every one of §39.14, §39.18 and §39.19 would have +been a parity failure had parity ever been checked. Keeping the tests would +have meant either reverting the fixes or explaining the failures away, and the +second is how a check becomes decorative. + +**Removed:** `src/homemaker_layout/oracle.py`, `tests/test_oracle.py`, the two +parity tests and their fixture machinery in `tests/test_dom_corpus.py`, +`innerloop.OracleEvaluator` and its `use_native` / `urb_root` plumbing, the +same plumbing through `driver`, and fourteen `experiments/` scripts that could +only run against Perl — `accept_innerloop`, `bakeoff_innerloop`, +`bakeoff_native`, `bench_batch_oracle`, `benchmark_vs_urbevolve`, +`genome_parity`, `leaf_parity`, `operator_locality`, `optimize_fullfitness`, +`rebaseline_no_occlusion`, `refine_sweep`, `resolve_ratios`, `run_search`, +`sweep_failtypes`. Several are cited in earlier sections as the evidence for +their measurements; those citations now point into git history rather than the +working tree, which is the honest state — they had been unrunnable since the +oracle root (`/home/bruno/src/urb`) stopped being present. `run_search` is +superseded by `run_search_scaled`, which does the same job natively. + +**Kept:** `dump_areas.pl`/`.py`, which validate *geometry* against Urb (§4.1) +rather than fitness, and the prose in `fitness_cmd.py` and `dom.py` explaining +why the `.score`/`.fails` formats are shaped as they are. Provenance is worth +keeping; a dead code path is not. + +**What this changes about reading the objective.** "Urb did it this way" is no +longer an argument that a constant is right — but §39.16 is the standing +counterweight, and it cuts the other way: the crinkliness target *was* right, +and twice looked wrong only because the code reading it was misunderstood. The +test is now the same as for anything else. Does the number have a stated +derivation, does it survive measurement, and does the objective it belongs to +say what its author meant? Inheritance is neither evidence for nor against. + +Next: `homemaker-py-hxi` — circulation at 0.07 return against a room's 0.66, +which the owner has ruled needs fixing — and the corpus re-baseline +(`homemaker-py-bk9`), which §39.19 made necessary and which every future number +depends on. diff --git a/experiments/accept_innerloop.py b/experiments/accept_innerloop.py deleted file mode 100644 index 5f922ec..0000000 --- a/experiments/accept_innerloop.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -"""Acceptance run for the geometry inner loop (homemaker-py-1p0). - -Gate (DESIGN.md §4.5, issue acceptance criteria): reproduce or exceed the -Nelder-Mead diagnostic gains — x1.24 / x1.67 / x1.59, no new failures — on -2f45907, candidate-002 and c964435, using the batched compass search. - -Usage: accept_innerloop.py [budget] [method] (default: 200 oracle evals, cma) -""" - -from __future__ import annotations - -import shutil -import sys -import tempfile -import time -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, innerloop, oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples" / "programme-house" - -# name -> reference x-gain. Current bars are the occlusion-disabled re-baseline -# (homemaker-py-gp2): the deterministic-seed CMA run at budget 400 under -# URB_NO_OCCLUSION=1 — set that env var to reproduce. (The original §4.5 -# Nelder-Mead bars, flag-off, were 1.24 / 1.67 / 1.59; the inner loop met them -# within noise, homemaker-py-1p0.) -GATE = { - "2f45907abd9accac2a124d311732f749.dom": 1.63, - "candidate-002.dom": 1.70, - "c964435454c459f86c3ed9a5a7621132.dom": 1.68, -} - -# Bars are single optimiser draws and per-run variance brackets them -# (candidate-002 drew 0.0117-0.0160 against a 0.0123 flag-off bar). -# Reproduction within 1% counts as met — decision approved 2026-06-12 -# (homemaker-py-1p0); chasing the last fraction with seed rolls would be -# cherry-picking. -NOISE_TOL = 0.99 - - -def main() -> int: - budget = int(sys.argv[1]) if len(sys.argv) > 1 else 200 - method = sys.argv[2] if len(sys.argv) > 2 else "cma" - print(f"method={method} budget={budget}") - all_ok = True - with tempfile.TemporaryDirectory(prefix="accept_innerloop_") as tmp: - scratch = Path(tmp) - shutil.copy(EX / "patterns.config", scratch) - for name, bar in GATE.items(): - # Baseline = the UNMODIFIED original file, exactly as §4.5 measured - # it. The inner loop's own x0 score is the equal-offset *projection* - # of the original (a==b forced, clipped), which is lower for legacy - # designs with unequal cuts — gains must not be measured from there. - s_orig = oracle.score(shutil.copy(EX / name, scratch), URB) - - root = dom.load(str(EX / name)) - t0 = time.perf_counter() - r = innerloop.optimise(root, EX, budget=budget, method=method, urb_root=URB) - dt = time.perf_counter() - t0 - gain = r.fitness / s_orig.fitness if s_orig.fitness else float("inf") - ok = gain >= bar * NOISE_TOL and r.n_fails <= s_orig.n_fails - all_ok &= ok - print( - f"{name:42s} dof={len(r.x):2d} " - f"orig={s_orig.fitness:.6g}(fails {s_orig.n_fails}) " - f"projected x0={r.x0_fitness:.6g}(fails {r.x0_n_fails}) " - f"opt={r.fitness:.6g}(fails {r.n_fails}) " - f"x{gain:.2f} (bar x{bar:.2f}) " - f"{r.n_evals} evals / {r.n_oracle_calls} oracle calls / {dt:.0f}s " - f"{'PASS' if ok else 'FAIL'}", - flush=True, - ) - print("\nGATE " + ("PASSED" if all_ok else "FAILED")) - return 0 if all_ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/bakeoff_innerloop.py b/experiments/bakeoff_innerloop.py deleted file mode 100644 index 8fb6d3c..0000000 --- a/experiments/bakeoff_innerloop.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python3 -"""Inner-loop optimiser bake-off at equal oracle budgets (homemaker-py-d0s). - -DESIGN.md §7 Phase 1(b), §8.3. DOF is only ≈ rooms−1 (6–7 on the corpus), so -the question is fitness gained per oracle evaluation, not asymptotic power. -Candidates: - - nm multi-start Nelder-Mead (scipy) — the §4.5 diagnostic optimiser. - Inherently sequential: ONE dom per oracle call, so the Perl - startup never amortises (§4.6). - cma multi-phase CMA-ES (innerloop.cma_search), one batched oracle - call per generation. - compass single-start batched compass search with pattern moves + random - augmentation (innerloop.compass_search). - compass-ms multi-start compass: budget split across restarts (x0 first, - then random starts), global best kept. - -Protocol: cold start from each file's equal-offset projection (x_current), -one run per (method, file, seed), best-so-far recorded after every oracle -call. Fitness-at-budget-B is read off the trace (evals ≤ B), so methods are -compared at exactly equal budgets regardless of batch granularity; checkpoint -budgets bracket the driver's real operating points (child_budget=80 warm, -seed_budget=200 cold). Wall-clock and oracle-invocation counts are recorded -per run. - -Runs under URB_NO_OCCLUSION=1 (set by this script — the gp2 re-baseline flag; -all benchmarks must use it). - -Usage: python3 experiments/bakeoff_innerloop.py [budget] [out.json] - (defaults: budget 200, experiments/bakeoff_innerloop.json) -""" - -from __future__ import annotations - -import json -import os -import shutil -import sys -import tempfile -import time -from pathlib import Path - -import numpy as np - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, innerloop, oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples" / "programme-house" - -FILES = ( - "2f45907abd9accac2a124d311732f749.dom", - "candidate-002.dom", - "c964435454c459f86c3ed9a5a7621132.dom", -) -SEEDS = (0, 1, 2) -CHECKPOINTS = (40, 80, 120, 200) - - -class TracingEvaluator(innerloop.OracleEvaluator): - """Records (cumulative evals, batch-best fitness) after every oracle call.""" - - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - self.trace: list[tuple[int, float]] = [] - - def evaluate(self, xs): - scores = super().evaluate(xs) - self.trace.append((self.n_evals, max(s.fitness for s in scores))) - return scores - - def best_at(self, budget: int) -> float: - vals = [f for n, f in self.trace if n <= budget] - return max(vals) if vals else float("nan") - - -class _BudgetExhausted(Exception): - pass - - -def nm_search(ev, x0, budget=200, seed=0): - """Multi-start Nelder-Mead: x0 first, random restarts until the budget is - spent. Sequential — every evaluation is its own oracle invocation.""" - from scipy.optimize import minimize - - rng = np.random.default_rng(seed) - n = len(x0) - x = np.clip(np.asarray(x0, dtype=float), innerloop._EPS, 1 - innerloop._EPS) - s = ev.evaluate([x])[0] - best = innerloop.Result( - x=x.copy(), fitness=s.fitness, n_fails=s.n_fails, fail_lines=s.fail_lines, - x0_fitness=s.fitness, x0_n_fails=s.n_fails, n_evals=0, n_oracle_calls=0, - ) - - def f(xi): - if ev.n_evals >= budget: - raise _BudgetExhausted - sc = ev.evaluate([np.asarray(xi, dtype=float)])[0] - if sc.fitness > best.fitness: - best.x = np.asarray(xi, dtype=float).copy() - best.fitness = sc.fitness - best.n_fails = sc.n_fails - best.fail_lines = sc.fail_lines - return -sc.fitness - - start = x.copy() - while ev.n_evals < budget: - try: - minimize( - f, start, method="Nelder-Mead", - bounds=[(innerloop._EPS, 1 - innerloop._EPS)] * n, - options={"maxfev": budget - ev.n_evals, "xatol": 1e-3, "fatol": 1e-10}, - ) - except _BudgetExhausted: - break - start = rng.uniform(0.1, 0.9, n) # restart - - best.n_evals = ev.n_evals - best.n_oracle_calls = ev.n_oracle_calls - return best - - -def compass_ms_search(ev, x0, budget=200, seed=0, n_starts=3): - """Multi-start compass: budget split evenly; first start is x0, the rest - random. compass_search counts against ev.n_evals, so phase budgets are - cumulative caps.""" - rng = np.random.default_rng(seed) - n = len(x0) - best = None - for phase in range(n_starts): - phase_end = ev.n_evals + (budget - ev.n_evals) // (n_starts - phase) - start = np.asarray(x0, dtype=float) if phase == 0 else rng.uniform(0.1, 0.9, n) - r = innerloop.compass_search(ev, start, budget=phase_end, seed=seed + phase) - if best is None or r.fitness > best.fitness: - keep_x0 = best.x0_fitness if best is not None else r.x0_fitness - keep_x0f = best.x0_n_fails if best is not None else r.x0_n_fails - best = r - best.x0_fitness, best.x0_n_fails = keep_x0, keep_x0f - if ev.n_evals >= budget: - break - best.n_evals = ev.n_evals - best.n_oracle_calls = ev.n_oracle_calls - return best - - -METHODS = { - "nm": nm_search, - "cma": innerloop.cma_search, - "compass": innerloop.compass_search, - "compass-ms": compass_ms_search, -} - - -def main() -> int: - budget = int(sys.argv[1]) if len(sys.argv) > 1 else 200 - out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else ( - Path(__file__).parent / "bakeoff_innerloop.json") - os.environ["URB_NO_OCCLUSION"] = "1" - checkpoints = [c for c in CHECKPOINTS if c <= budget] or [budget] - if checkpoints[-1] != budget: - checkpoints.append(budget) - - # Baselines: the UNMODIFIED originals (gains measured from there, not from - # the equal-offset projection — accept_innerloop.py convention). - orig: dict[str, oracle.Score] = {} - with tempfile.TemporaryDirectory(prefix="bakeoff_orig_") as tmp: - shutil.copy(EX / "patterns.config", tmp) - for name in FILES: - orig[name] = oracle.score(shutil.copy(EX / name, tmp), URB) - - runs = [] - for name in FILES: - for method in METHODS: - for seed in SEEDS: - root = dom.load(str(EX / name)) - with TracingEvaluator(root, EX, URB) as ev: - x0 = ev.x_current - t0 = time.perf_counter() - r = METHODS[method](ev, x0, budget=budget, seed=seed) - dt = time.perf_counter() - t0 - run = { - "file": name, "method": method, "seed": seed, - "dof": len(x0), - "orig_fitness": orig[name].fitness, - "orig_n_fails": orig[name].n_fails, - "x0_fitness": r.x0_fitness, "x0_n_fails": r.x0_n_fails, - "best_at": {str(c): ev.best_at(c) for c in checkpoints}, - "final_fitness": r.fitness, "final_n_fails": r.n_fails, - "n_evals": ev.n_evals, "n_oracle_calls": ev.n_oracle_calls, - "wall_s": dt, - } - runs.append(run) - gains = " ".join( - f"@{c}:x{run['best_at'][str(c)] / orig[name].fitness:.2f}" - for c in checkpoints) - print( - f"{name[:12]:12s} {method:10s} seed={seed} {gains} " - f"fails {orig[name].n_fails}->{r.n_fails} " - f"{ev.n_evals}ev/{ev.n_oracle_calls}calls {dt:.0f}s", - flush=True, - ) - - out_path.write_text(json.dumps( - {"budget": budget, "checkpoints": checkpoints, "runs": runs}, indent=1)) - print(f"\nwrote {out_path}") - - # Summary: mean gain over original at each checkpoint, mean s/eval. - print(f"\n{'method':10s} " + "".join(f"{'x@' + str(c):>8s}" for c in checkpoints) - + f"{'s/eval':>8s}{'calls':>7s}{'fails+':>7s}") - for method in METHODS: - rs = [r for r in runs if r["method"] == method] - cols = "" - for c in checkpoints: - g = np.mean([r["best_at"][str(c)] / r["orig_fitness"] for r in rs]) - cols += f"{g:8.2f}" - spe = np.mean([r["wall_s"] / r["n_evals"] for r in rs]) - calls = np.mean([r["n_oracle_calls"] for r in rs]) - newf = sum(r["final_n_fails"] > r["orig_n_fails"] for r in rs) - print(f"{method:10s} {cols}{spe:8.2f}{calls:7.0f}{newf:7d}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/bakeoff_native.py b/experiments/bakeoff_native.py deleted file mode 100644 index 3fc701a..0000000 --- a/experiments/bakeoff_native.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -"""Inner-loop optimiser bake-off with native fitness (homemaker-py-d6d). - -Re-runs the Phase 1 oracle bakeoff (bakeoff_innerloop.py / homemaker-py-d0s) -using the native Python fitness evaluator instead of the Perl oracle. The -oracle's 1 s/eval startup cost previously penalised sequential methods (NM); -with native fitness at ~70 evals/s that constraint is gone. - -Candidates: nm, cma, compass, compass-ms (same as Phase 1). -Protocol: cold start from each file's equal-offset projection, one run per -(method, file, seed), best-so-far traced after every eval. Gains measured -relative to native baseline (native score of the unmodified original). - -Usage: python3 experiments/bakeoff_native.py [budget] [out.json] - (defaults: budget 200, experiments/bakeoff_native.json) -""" - -from __future__ import annotations - -import json -import os -import sys -import time -from pathlib import Path - -import numpy as np - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, fitness as fit_mod, innerloop, solver # noqa: E402 - -EX = Path("/home/bruno/src/urb/examples/programme-house") - -FILES = ( - "2f45907abd9accac2a124d311732f749.dom", - "candidate-002.dom", - "c964435454c459f86c3ed9a5a7621132.dom", -) -SEEDS = (0, 1, 2) -CHECKPOINTS = (40, 80, 120, 200) - - -class NativeTracingEvaluator(innerloop.NativeEvaluator): - """NativeEvaluator that records (cumulative evals, batch-best fitness).""" - - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - self.trace: list[tuple[int, float]] = [] - - def evaluate(self, xs): - scores = super().evaluate(xs) - self.trace.append((self.n_evals, max(s.fitness for s in scores))) - return scores - - def best_at(self, budget: int) -> float: - vals = [f for n, f in self.trace if n <= budget] - return max(vals) if vals else float("nan") - - -class _BudgetExhausted(Exception): - pass - - -def nm_search(ev, x0, budget=200, seed=0): - """Multi-start Nelder-Mead: x0 first, random restarts until budget spent.""" - from scipy.optimize import minimize - - rng = np.random.default_rng(seed) - n = len(x0) - x = np.clip(np.asarray(x0, dtype=float), innerloop._EPS, 1 - innerloop._EPS) - s = ev.evaluate([x])[0] - best = innerloop.Result( - x=x.copy(), fitness=s.fitness, n_fails=s.n_fails, fail_lines=s.fail_lines, - x0_fitness=s.fitness, x0_n_fails=s.n_fails, n_evals=0, n_oracle_calls=0, - ) - - def f(xi): - if ev.n_evals >= budget: - raise _BudgetExhausted - sc = ev.evaluate([np.asarray(xi, dtype=float)])[0] - if sc.fitness > best.fitness: - best.x = np.asarray(xi, dtype=float).copy() - best.fitness = sc.fitness - best.n_fails = sc.n_fails - best.fail_lines = sc.fail_lines - return -sc.fitness - - start = x.copy() - while ev.n_evals < budget: - try: - minimize( - f, start, method="Nelder-Mead", - bounds=[(innerloop._EPS, 1 - innerloop._EPS)] * n, - options={"maxfev": budget - ev.n_evals, "xatol": 1e-3, "fatol": 1e-10}, - ) - except _BudgetExhausted: - break - start = rng.uniform(0.1, 0.9, n) - - - best.n_evals = ev.n_evals - best.n_oracle_calls = ev.n_oracle_calls - return best - - -def compass_ms_search(ev, x0, budget=200, seed=0, n_starts=3): - """Multi-start compass: budget split evenly; first start is x0.""" - rng = np.random.default_rng(seed) - n = len(x0) - best = None - for phase in range(n_starts): - phase_end = ev.n_evals + (budget - ev.n_evals) // (n_starts - phase) - start = np.asarray(x0, dtype=float) if phase == 0 else rng.uniform(0.1, 0.9, n) - r = innerloop.compass_search(ev, start, budget=phase_end, seed=seed + phase) - if best is None or r.fitness > best.fitness: - keep_x0 = best.x0_fitness if best is not None else r.x0_fitness - keep_x0f = best.x0_n_fails if best is not None else r.x0_n_fails - best = r - best.x0_fitness, best.x0_n_fails = keep_x0, keep_x0f - if ev.n_evals >= budget: - break - best.n_evals = ev.n_evals - best.n_oracle_calls = ev.n_oracle_calls - return best - - -METHODS = { - "nm": nm_search, - "cma": innerloop.cma_search, - "compass": innerloop.compass_search, - "compass-ms": compass_ms_search, -} - - -def native_baseline(path: Path) -> innerloop._NativeScore: - """Native fitness of the unmodified file.""" - root = dom.load(str(path)) - conf, cost = fit_mod.load_config(EX) - fit = fit_mod.Fitness(conf, cost) - score, fails = fit.score_with_fails(root) - return innerloop._NativeScore(fitness=score, fail_lines=tuple(fails)) - - -def main() -> int: - budget = int(sys.argv[1]) if len(sys.argv) > 1 else 200 - out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else ( - Path(__file__).parent / "bakeoff_native.json") - os.environ["URB_NO_OCCLUSION"] = "1" - checkpoints = [c for c in CHECKPOINTS if c <= budget] or [budget] - if checkpoints[-1] != budget: - checkpoints.append(budget) - - orig: dict[str, innerloop._NativeScore] = {} - for name in FILES: - orig[name] = native_baseline(EX / name) - - runs = [] - for name in FILES: - for method in METHODS: - for seed in SEEDS: - root = dom.load(str(EX / name)) - ev = NativeTracingEvaluator(root, EX) - x0 = ev.x_current - t0 = time.perf_counter() - r = METHODS[method](ev, x0, budget=budget, seed=seed) - dt = time.perf_counter() - t0 - run = { - "file": name, "method": method, "seed": seed, - "dof": len(x0), - "orig_fitness": orig[name].fitness, - "orig_n_fails": orig[name].n_fails, - "x0_fitness": r.x0_fitness, "x0_n_fails": r.x0_n_fails, - "best_at": {str(c): ev.best_at(c) for c in checkpoints}, - "final_fitness": r.fitness, "final_n_fails": r.n_fails, - "n_evals": ev.n_evals, "n_oracle_calls": ev.n_oracle_calls, - "wall_s": dt, - } - runs.append(run) - gains = " ".join( - f"@{c}:x{run['best_at'][str(c)] / orig[name].fitness:.2f}" - for c in checkpoints) - print( - f"{name[:12]:12s} {method:10s} seed={seed} {gains} " - f"fails {orig[name].n_fails}->{r.n_fails} " - f"{ev.n_evals}ev {dt:.1f}s", - flush=True, - ) - - out_path.write_text(json.dumps( - {"budget": budget, "checkpoints": checkpoints, "runs": runs}, indent=1)) - print(f"\nwrote {out_path}") - - print(f"\n{'method':10s} " + "".join(f"{'x@' + str(c):>8s}" for c in checkpoints) - + f"{'s/eval':>8s}{'fails+':>7s}") - for method in METHODS: - rs = [r for r in runs if r["method"] == method] - cols = "" - for c in checkpoints: - g = np.mean([r["best_at"][str(c)] / r["orig_fitness"] for r in rs]) - cols += f"{g:8.3f}" - spe = np.mean([r["wall_s"] / r["n_evals"] for r in rs]) - newf = sum(r["final_n_fails"] > r["orig_n_fails"] for r in rs) - print(f"{method:10s} {cols}{spe:8.4f}{newf:7d}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/bench_batch_oracle.py b/experiments/bench_batch_oracle.py deleted file mode 100644 index 9e9af55..0000000 --- a/experiments/bench_batch_oracle.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -"""Validate and benchmark the batched oracle (homemaker-py-av5). - -Copies the 35-file programme-house corpus into a scratch directory, scores -every file via single-file oracle calls and again via one batched invocation, -then checks the per-file fitness and failure sets are identical and reports -measured s/dom for both modes. -""" - -from __future__ import annotations - -import math -import shutil -import sys -import tempfile -import time -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -CORPUS = URB / "examples" / "programme-house" - - -def main() -> int: - with tempfile.TemporaryDirectory(prefix="bench_batch_") as tmp: - scratch = Path(tmp) - shutil.copy(CORPUS / "patterns.config", scratch) - doms = sorted(CORPUS.glob("*.dom")) - paths = [shutil.copy(d, scratch) for d in doms] - paths = [Path(p) for p in paths] - print(f"{len(paths)} corpus files -> {scratch}") - - t0 = time.perf_counter() - singles = [oracle.score(p, URB) for p in paths] - t_single = time.perf_counter() - t0 - - t0 = time.perf_counter() - batch = oracle.score_batch(paths, URB) - t_batch = time.perf_counter() - t0 - - # Urb's score is nondeterministic at the ~1 ULP level (Perl hash-order - # summation), so compare fitness with a tight relative tolerance; any - # real semantic difference (e.g. occlusion handling) would be far larger. - mismatches = 0 - for p, s, b in zip(paths, singles, batch): - if not math.isclose(s.fitness, b.fitness, rel_tol=1e-12) or s.fail_lines != b.fail_lines: - mismatches += 1 - print(f"MISMATCH {p.name}: single {s.fitness:.12g} ({s.n_fails} fails) " - f"vs batch {b.fitness:.12g} ({b.n_fails} fails)") - print(f" single-only: {sorted(set(s.fail_lines) - set(b.fail_lines))}") - print(f" batch-only: {sorted(set(b.fail_lines) - set(s.fail_lines))}") - - n = len(paths) - print(f"\nsingle-file: {t_single:.2f} s total, {t_single / n:.3f} s/dom") - print(f"batched: {t_batch:.2f} s total, {t_batch / n:.3f} s/dom") - print(f"speedup: x{t_single / t_batch:.2f}") - if mismatches: - print(f"\nFAIL: {mismatches}/{n} files differ between single and batch") - return 1 - print(f"\nOK: all {n} files identical (fitness to 1e-12 rel, exact failure set) in both modes") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/benchmark_vs_urbevolve.py b/experiments/benchmark_vs_urbevolve.py deleted file mode 100644 index 3a37686..0000000 --- a/experiments/benchmark_vs_urbevolve.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Phase-2 gate: memetic loop vs urb-evolve at equal oracle-eval budgets -(homemaker-py-way). - -Protocol: 3 seed designs x 2000-evaluation runs; the 1000-eval tier is read -from each run's own best-so-far log (no extra oracle time). urb-evolve runs -at two population sizes (its default 128, which only allows ~16 generations -at this budget, and 16, which allows ~130) and gets credit for its better -one. urb-evolve counts evaluations via the MAX_EVALS patch; the memetic -driver accounts natively. Both run under URB_NO_OCCLUSION=1 with the same -patterns.config; every final design is re-scored through urb-fitness.pl as -the common deterministic yardstick. - - URB_NO_OCCLUSION=1 python3 experiments/benchmark_vs_urbevolve.py [budget] -""" - -from __future__ import annotations - -import os -import re -import shutil -import subprocess -import sys -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples" / "programme-house" -HERE = Path(__file__).resolve().parent -WORK = HERE.parent / "scratch" / "benchmark" - -SEEDS = ["init.dom", "c964435454c459f86c3ed9a5a7621132.dom", - "2f45907abd9accac2a124d311732f749.dom"] -CHECKPOINT_FRACTION = 0.5 - - -def run_homemaker(seed: str, budget: int, cell: Path) -> dict: - out = cell / "best.dom" - proc = subprocess.run( - [sys.executable, str(HERE / "run_search.py"), str(budget), "0", - str(EX / seed), str(out)], - capture_output=True, text=True, env={**os.environ, "URB_NO_OCCLUSION": "1"}, - ) - (cell / "run.log").write_text(proc.stdout + proc.stderr) - series = [(int(m[1]), float(m[2])) for m in - re.finditer(r"\[\s*(\d+) evals\] best ([\d.e-]+)", proc.stdout)] - return {"series": series, "best_dom": out, "log_ok": proc.returncode == 0} - - -def run_urbevolve(seed: str, budget: int, cell: Path, pop: int) -> dict: - shutil.copy(EX / "patterns.config", cell) - shutil.copy(EX / seed, cell) - proc = subprocess.run( - ["perl", f"-I{URB}/lib", str(URB / "bin" / "urb-evolve.pl"), seed], - cwd=cell, capture_output=True, text=True, - env={**os.environ, "URB_NO_OCCLUSION": "1", "MAX_EVALS": str(budget), - "MAX_POP": str(pop), "MAX_ITERATIONS": "100000"}, - ) - (cell / "run.log").write_text(proc.stderr) - series = [] - evals = None - for line in proc.stderr.splitlines(): - if m := re.search(r"Evals: (\d+)", line): - evals = int(m[1]) - elif (m := re.search(r"Fitness: ([\d.e-]+)", line)) and evals is not None: - series.append((evals, float(m[1]))) - out_hash = proc.stdout.strip().split()[-1] if proc.stdout.strip() else None - best = cell / f"{out_hash}.dom" if out_hash else None - return {"series": series, "best_dom": best, - "log_ok": proc.returncode == 0 and best is not None and best.exists()} - - -def best_at(series: list[tuple[int, float]], budget: int) -> float: - vals = [f for e, f in series if e <= budget] - return max(vals) if vals else float("nan") - - -def main() -> int: - budget = int(sys.argv[1]) if len(sys.argv) > 1 else 2000 - checkpoint = int(budget * CHECKPOINT_FRACTION) - if WORK.exists(): - shutil.rmtree(WORK) - - cells = [] - for seed in SEEDS: - cells.append((seed, "memetic", run_homemaker, {})) - cells.append((seed, "urb-evolve p16", run_urbevolve, {"pop": 16})) - cells.append((seed, "urb-evolve p128", run_urbevolve, {"pop": 128})) - - def run_cell(args): - seed, label, fn, kw = args - cell = WORK / f"{seed.split('.')[0][:8]}_{label.replace(' ', '_')}" - cell.mkdir(parents=True, exist_ok=True) - print(f"start: {seed} / {label}", flush=True) - res = fn(seed, budget, cell, **kw) - print(f"done: {seed} / {label} ({len(res['series'])} log points)", flush=True) - return (seed, label), res - - with ThreadPoolExecutor(max_workers=3) as pool: - results = dict(pool.map(run_cell, cells)) - - # common yardstick: re-score every final design through urb-fitness.pl - for key, res in results.items(): - if res["best_dom"] and Path(res["best_dom"]).exists(): - s = oracle.score(res["best_dom"], URB) - res["final"], res["fails"] = s.fitness, s.n_fails - else: - res["final"], res["fails"] = float("nan"), -1 - - print(f"\n{'seed':10s} {'system':16s} {'best@' + str(checkpoint):>12s} " - f"{'final@' + str(budget):>12s} {'fails':>6s}") - verdicts = [] - for seed in SEEDS: - rows = {label: results[(seed, label)] for label in - ("memetic", "urb-evolve p16", "urb-evolve p128")} - for label, res in rows.items(): - print(f"{seed[:10]:10s} {label:16s} {best_at(res['series'], checkpoint):12.6g} " - f"{res['final']:12.6g} {res['fails']:6d}") - hm = rows["memetic"]["final"] - urb = max(rows["urb-evolve p16"]["final"], rows["urb-evolve p128"]["final"]) - verdicts.append(hm > urb) - print(f"{'':10s} -> memetic {'BEATS' if hm > urb else 'LOSES TO'} " - f"urb-evolve ({hm:.6g} vs {urb:.6g})") - print(f"\nGATE: memetic wins {sum(verdicts)}/{len(verdicts)} seeds " - f"-> {'GO' if all(verdicts) else 'REVIEW'}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/diag_2f45907.py b/experiments/diag_2f45907.py index f6c4265..9a04c6e 100644 --- a/experiments/diag_2f45907.py +++ b/experiments/diag_2f45907.py @@ -21,7 +21,6 @@ from scipy.optimize import minimize sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from homemaker_layout import dom, innerloop # noqa: E402 -URB = Path("/home/bruno/src/urb") EX = URB / "examples" / "programme-house" NAME = "2f45907abd9accac2a124d311732f749.dom" @@ -52,7 +51,7 @@ def main() -> None: f"({ev.n_evals - n0} evals)", flush=True) root = dom.load(str(EX / NAME)) - r = innerloop.optimise(root, EX, budget=200, method="cma", sigmas=(0.05,), urb_root=URB) + r = innerloop.optimise(root, EX, budget=200, method="cma", sigmas=(0.05,)) print(f"CMA sigma 0.05: {r.fitness:.6g} fails {r.n_fails} ({r.n_evals} evals)") diff --git a/experiments/diag_depth_balance.py b/experiments/diag_depth_balance.py index 2c1e595..fa4fc3d 100644 --- a/experiments/diag_depth_balance.py +++ b/experiments/diag_depth_balance.py @@ -129,7 +129,7 @@ def _measure(fit, pdir, seed_root, reqs, types, s, balanced, sharing, factor): after_tree = copy.deepcopy(topo) with _force_sharing(sharing): innerloop.optimise(after_tree, str(pdir), x0=None, budget=BUDGET, - method="nm", use_native=True) + method="nm") _s2, fails2 = fit.score_with_fails(copy.deepcopy(after_tree)) after = {"n_leaves": n_leaves, "total": len(fails2), **_bucket(fails2), **_maldist(after_tree, fit, reqs)} diff --git a/experiments/diag_leaf_sharing.py b/experiments/diag_leaf_sharing.py index 7656286..829c406 100644 --- a/experiments/diag_leaf_sharing.py +++ b/experiments/diag_leaf_sharing.py @@ -105,7 +105,7 @@ def _measure(fit, pdir, seed_root, reqs, types, s, sharing, factor): after_tree = copy.deepcopy(topo) with _force_sharing(sharing): innerloop.optimise(after_tree, str(pdir), x0=None, budget=BUDGET, - method="nm", use_native=True) + method="nm") _s2, fails2 = fit.score_with_fails(copy.deepcopy(after_tree)) after = {"n_leaves": n_leaves, "total": len(fails2), **_bucket(fails2)} return before, after diff --git a/experiments/diag_slack_localization.py b/experiments/diag_slack_localization.py index bd7fc82..0200c04 100644 --- a/experiments/diag_slack_localization.py +++ b/experiments/diag_slack_localization.py @@ -134,7 +134,7 @@ def _run_seed(pdir, fit, reqs, types, seed_root, s): after_tree = copy.deepcopy(topo) r = innerloop.optimise(after_tree, str(pdir), x0=None, budget=BUDGET, - method="nm", use_native=True) + method="nm") after = _measure(after_tree, fit, reqs) after["n_evals"] = r.n_evals return before, after, _leaf_ratios(topo, fit, reqs) diff --git a/experiments/dump_leaf_quality.pl b/experiments/dump_leaf_quality.pl deleted file mode 100644 index 500214e..0000000 --- a/experiments/dump_leaf_quality.pl +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/perl -# Parity oracle for homemaker-py-gnw: per-leaf quality factors and per-storey -# cost/value from Urb's programme-driven fitness. -# -# Stock urb-fitness.pl cannot emit the per-leaf DEBUG lines: Leaf.pm/Storey.pm -# copy $Urb::Dom::Fitness::DEBUG into a package var at *compile* time, before -# Fitness.pm's own `our $DEBUG = $ENV{DEBUG}` line has run, so the flag is -# permanently false. This wrapper sets those package vars after loading and -# also wraps process_storey to print each storey's cost/value subtotal. -# -# Usage (from the corpus directory, URB_NO_OCCLUSION required): -# cd examples/programme-house -# URB_NO_OCCLUSION=1 perl -I$URB/lib dump_leaf_quality.pl a.dom b.dom ... -# -# Output (stdout): -# === FILE -# ...Leaf.pm debug lines (' quality perpendicular: ' x7, then -# 'leaf level: L id: I type: T rate: R area: A width: W proportion: P') -# STOREY cost: value: -# INITIAL_COST -# FAIL (one per failure) -# SCORE - -use strict; -use warnings; -use 5.010; -use Urb::Dom; -use Urb::Dom::Fitness; -use Urb::Dom::Fitness::ProgrammeDriven; -use YAML; -use Carp; - -die "URB_NO_OCCLUSION=1 required: native fitness ports simple crinkliness only\n" - unless $ENV{URB_NO_OCCLUSION}; - -# Re-enable the per-leaf/storey debug gates (see header) and route everything -# to STDOUT so output interleaves deterministically. -$Urb::Dom::Fitness::Leaf::DEBUG = 1; -$Urb::Dom::Fitness::Storey::DEBUG = 1; -$Urb::Dom::Fitness::DEBUG = 1; -{ - no warnings 'redefine'; - *Urb::Dom::Fitness::debug = sub { print join(' ', @_), "\n"; return 1 }; - *Urb::Dom::Fitness::Base::debug = sub { - my @text = @_; - shift @text if ref $text[0]; # method form: drop $self - print join(' ', @text), "\n"; - return 1; - }; -} - -my $orig_process_storey = \&Urb::Dom::Fitness::Storey::process_storey; -{ - no warnings 'redefine'; - *Urb::Dom::Fitness::Storey::process_storey = sub { - my @args = @_; - my $result = $orig_process_storey->(@args); - printf "STOREY %s cost: %.17g value: %.17g\n", - $args[5], $result->{cost}, $result->{value}; - return $result; - }; -} - -# Config loading exactly as urb-fitness.pl (project-level then local). -my $config = undef; -for my $path ('../patterns.config', 'patterns.config') -{ - next unless -e $path; - my $config_temp = YAML::LoadFile ($path); - $config->{$_} = $config_temp->{$_} for keys %{$config_temp}; -} -my $costs = undef; -for my $path ('../costs.config', 'costs.config') -{ - next unless -e $path; - my $costs_temp = YAML::LoadFile ($path); - $costs->{$_} = $costs_temp->{$_} for keys %{$costs_temp}; -} - -die "no spaces config: wrapper only supports programme-driven mode\n" - unless $config && exists $config->{spaces}; -my $assessor = Urb::Dom::Fitness::ProgrammeDriven->new ($config, $costs); - -for my $path_yaml (@ARGV) -{ - my @items = YAML::LoadFile ($path_yaml); - next unless scalar @items; - my $dom = Urb::Dom->new; - $dom->Deserialise (@items); - - say "=== FILE $path_yaml"; - printf "INITIAL_COST %.17g\n", $assessor->Cost ('plot') * $dom->Area; - my $score = $assessor->_apply ($dom); - say "FAIL $_" for ($dom->Failures); - printf "SCORE %.17g\n", $score; - $dom->DESTROY; -} - -0; diff --git a/experiments/genome_parity.py b/experiments/genome_parity.py deleted file mode 100644 index 43a3ef4..0000000 --- a/experiments/genome_parity.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -"""Genome round-trip fitness parity (homemaker-py-k2g acceptance). - -decode(encode(load(f))) canonicalises dead fields (inherited-cut divisions, -below-linked rotations, internal types) that the corpus carries in drifted -form. This scores every original and its round-tripped twin through the -oracle and demands identical fitness (1e-12 rel, Urb's ~1-ULP jitter) and -identical failure sets. - -Run under the go-forward fitness: URB_NO_OCCLUSION=1 python3 experiments/genome_parity.py -""" - -from __future__ import annotations - -import math -import shutil -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, genome, oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -CORPUS = URB / "examples" / "programme-house" - - -def main() -> int: - with tempfile.TemporaryDirectory(prefix="genome_parity_") as tmp: - scratch = Path(tmp) - shutil.copy(CORPUS / "patterns.config", scratch) - originals, twins = [], [] - for src in sorted(CORPUS.glob("*.dom")): - o = Path(shutil.copy(src, scratch)) - t = scratch / ("rt_" + src.name) - dom.dump(genome.decode(genome.encode(dom.load(str(src)))), str(t)) - originals.append(o) - twins.append(t) - - s_orig = oracle.score_batch(originals, URB) - s_twin = oracle.score_batch(twins, URB) - - bad = 0 - for o, a, b in zip(originals, s_orig, s_twin): - if not math.isclose(a.fitness, b.fitness, rel_tol=1e-12) or a.fail_lines != b.fail_lines: - bad += 1 - print(f"MISMATCH {o.name}: {a.fitness:.17g} ({a.n_fails} fails) vs " - f"{b.fitness:.17g} ({b.n_fails} fails)") - n = len(originals) - print(f"{'FAIL' if bad else 'OK'}: {n - bad}/{n} files fitness-identical " - f"after genome round-trip") - return 1 if bad else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/leaf_parity.py b/experiments/leaf_parity.py deleted file mode 100644 index 85df780..0000000 --- a/experiments/leaf_parity.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -"""homemaker-py-gnw acceptance: per-leaf quality factors and per-storey -cost/value parity between homemaker.fitness and Urb's programme-driven fitness. - -Runs experiments/dump_leaf_quality.pl (URB_NO_OCCLUSION oracle with per-leaf -DEBUG re-enabled) over the corpus, parses its output, and diffs against the -native evaluation: 7 quality factors + rate + area per leaf, cost/value per -storey, initial (plot) cost, and the leaf-scope failure set. - -Usage: python3 experiments/leaf_parity.py [corpus_dir] -""" - -from __future__ import annotations - -import math -import re -import subprocess -import sys -from collections import defaultdict -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, fitness, graph # noqa: E402 - -URB = Path("/home/bruno/src/urb") -CORPUS = Path(sys.argv[1]) if len(sys.argv) > 1 else URB / "examples" / "programme-house" -WRAPPER = Path(__file__).resolve().parent / "dump_leaf_quality.pl" - -REL_TOL = 1e-9 - -# Failures emitted by the gnw scope (leaf quality + cost functions + the two -# per-leaf structural checks in process_storey). All carry a "level/" prefix — -# requiring it excludes building-level fails like 'no outside public access' -# (homemaker-py-hgg scope), which would otherwise match the ' access' suffix. -_LEAF_FAIL_RE = re.compile( - r"^\d+/.* (perpendicular|proportion|size|width|crinkliness|daylight|access" - r"|edge too long|unsupported covered outside|covered outside above ground)$" -) - -_FACTOR_RE = re.compile(r"^ quality (\w+): (\S+)$") -_LEAF_RE = re.compile( - r"^leaf level: (\d+) id: ([lr]*) type: (\S+) rate: (\S+) area: (\S+) " - r"width: (\S+) proportion: (\S+)$" -) -_STOREY_RE = re.compile(r"^STOREY (\d+) cost: (\S+) value: (\S+)$") - - -def run_oracle(files: list[str]) -> dict[str, dict]: - """Run the wrapper over all files; return per-file parsed records.""" - proc = subprocess.run( - ["perl", f"-I{URB}/lib", str(WRAPPER)] + files, - cwd=CORPUS, - env={"URB_NO_OCCLUSION": "1", "PATH": "/usr/bin:/bin"}, - capture_output=True, - text=True, - ) - if proc.returncode != 0: - sys.exit(f"wrapper failed:\n{proc.stderr}") - - out: dict[str, dict] = {} - rec = None - pending: dict[str, float] = {} - for line in proc.stdout.splitlines(): - if line.startswith("=== FILE "): - rec = out[line[len("=== FILE "):]] = { - "leaves": {}, "storeys": {}, "fails": [], "initial_cost": None, - "score": None, - } - pending = {} - continue - if rec is None: - continue - m = _FACTOR_RE.match(line) - if m: - pending[m.group(1)] = float(m.group(2)) - continue - m = _LEAF_RE.match(line) - if m: - level, lid = int(m.group(1)), m.group(2) - rec["leaves"][(level, lid)] = { - "type": m.group(3), - "rate": float(m.group(4)), - "area": float(m.group(5)), - "factors": pending, - } - pending = {} - continue - m = _STOREY_RE.match(line) - if m: - rec["storeys"][int(m.group(1))] = (float(m.group(2)), float(m.group(3))) - continue - if line.startswith("INITIAL_COST "): - rec["initial_cost"] = float(line.split()[1]) - elif line.startswith("FAIL "): - rec["fails"].append(line[len("FAIL "):]) - elif line.startswith("SCORE "): - rec["score"] = float(line.split()[1]) - return out - - -def native_eval(path: Path, fit: fitness.Fitness) -> dict: - root = dom.load(str(path)) - fails: list[str] = [] - fit.preprocess_building(root) - dom.merge_divided(root) - graphs = graph.build_graphs(root) - - rec: dict = {"leaves": {}, "storeys": {}, "fails": fails, - "initial_cost": fit.plot_cost(root)} - for li, lvl in enumerate(dom.levels(root)): - se = fit.process_storey(lvl, graphs[li], li, fails.append) - rec["storeys"][li] = (se.cost, se.value) - for le in se.leaves: - rec["leaves"][(le.level, le.id)] = { - "type": le.type, "rate": le.rate, "area": le.area, - "factors": le.factors, - } - return rec - - -def close(a: float, b: float) -> bool: - return math.isclose(a, b, rel_tol=REL_TOL, abs_tol=1e-12) - - -def main() -> int: - files = sorted(p.name for p in CORPUS.glob("*.dom")) - conf, cost = fitness.load_config(CORPUS) - fit = fitness.Fitness(conf, cost) - oracle = run_oracle(files) - - n_leaves = n_factors = 0 - bad: dict[str, list[str]] = defaultdict(list) - - for name in files: - o = oracle[name] - n = native_eval(CORPUS / name, fit) - - if not close(o["initial_cost"], n["initial_cost"]): - bad[name].append( - f"initial cost {o['initial_cost']} != {n['initial_cost']}") - - if set(o["leaves"]) != set(n["leaves"]): - bad[name].append( - f"leaf sets differ: oracle-only {sorted(set(o['leaves']) - set(n['leaves']))}, " - f"native-only {sorted(set(n['leaves']) - set(o['leaves']))}") - for key in sorted(set(o["leaves"]) & set(n["leaves"])): - ol, nl = o["leaves"][key], n["leaves"][key] - n_leaves += 1 - for fld in ("rate", "area"): - if not close(ol[fld], nl[fld]): - bad[name].append(f"leaf {key} {fld}: {ol[fld]} != {nl[fld]}") - if ol["type"] != nl["type"]: - bad[name].append(f"leaf {key} type: {ol['type']} != {nl['type']}") - for fac, ov in ol["factors"].items(): - n_factors += 1 - nv = nl["factors"].get(fac) - if nv is None or not close(ov, nv): - bad[name].append(f"leaf {key} {fac}: {ov} != {nv}") - - for li, (oc, ov) in o["storeys"].items(): - nc, nv = n["storeys"].get(li, (None, None)) - if nc is None or not close(oc, nc): - bad[name].append(f"storey {li} cost: {oc} != {nc}") - if nv is None or not close(ov, nv): - bad[name].append(f"storey {li} value: {ov} != {nv}") - - o_fails = sorted(f for f in o["fails"] if _LEAF_FAIL_RE.search(f)) - n_fails = sorted(n["fails"]) - if o_fails != n_fails: - bad[name].append( - f"leaf fails differ:\n oracle: {o_fails}\n native: {n_fails}") - - for name in sorted(bad): - print(f"MISMATCH {name}") - for msg in bad[name]: - print(f" {msg}") - print(f"\n{len(files)} files, {n_leaves} leaves, {n_factors} factors compared; " - f"{len(bad)} files with mismatches") - return 1 if bad else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/operator_locality.py b/experiments/operator_locality.py deleted file mode 100644 index 86e43bf..0000000 --- a/experiments/operator_locality.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -"""Operator validity + locality measurement (homemaker-py-nyb acceptance). - -For each operator: apply 5 seeded instances per corpus design, score every -child through the oracle (validity = scores without error), and report -locality as (a) mean relative fitness perturbation and (b) mean geometry -perturbation — the fraction of leaf rooms whose (type, footprint) changed. -High-locality operators keep both small, so warm-started inner loops stay -cheap (DESIGN.md §5). - -Run under the go-forward fitness: - URB_NO_OCCLUSION=1 python3 experiments/operator_locality.py -""" - -from __future__ import annotations - -import shutil -import sys -import tempfile -from collections import Counter -from pathlib import Path - -import numpy as np - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, genome, geometry, operators, oracle, programme # noqa: E402 - -URB = Path("/home/bruno/src/urb") -CORPUS = URB / "examples" / "programme-house" -FILES = ["2f45907abd9accac2a124d311732f749.dom", "candidate-002.dom", - "c964435454c459f86c3ed9a5a7621132.dom"] -SEEDS = range(5) - - -def leaf_signature(root: dom.Node) -> Counter: - sig = Counter() - for li, lvl in enumerate(dom.levels(root)): - for leaf in lvl.leaves(): - corners = tuple(tuple(round(c, 6) for c in geometry.coordinate(leaf, i)) - for i in range(4)) - sig[(li, leaf.type, corners)] += 1 - return sig - - -def geometry_perturbation(parent_sig: Counter, child: dom.Node) -> float: - child_sig = leaf_signature(child) - common = sum((parent_sig & child_sig).values()) - return 1.0 - common / max(parent_sig.total(), child_sig.total()) - - -def main() -> int: - types = sorted(programme.load_programme(str(CORPUS / "patterns.config"))) + ["C", "O"] - roots = {f: genome.decode(genome.encode(dom.load(str(CORPUS / f)))) for f in FILES} - - with tempfile.TemporaryDirectory(prefix="op_locality_") as tmp: - scratch = Path(tmp) - shutil.copy(CORPUS / "patterns.config", scratch) - - parents = {} - paths = [] - for f, root in roots.items(): - p = scratch / f - dom.dump(root, str(p)) - paths.append(p) - for f, s in zip(roots, oracle.score_batch(paths, URB)): - parents[f] = s - - jobs: list[tuple[str, str, dom.Node]] = [] # (op, desc, child) - for f, root in roots.items(): - for name, op in operators.MUTATIONS.items(): - for seed in SEEDS: - child, desc = op(root, np.random.default_rng(seed), types) - jobs.append((name, f, child)) - for seed in SEEDS: - ca, cb, _ = operators.crossover(roots[FILES[0]], roots[FILES[1]], - np.random.default_rng(seed)) - jobs.append(("crossover", FILES[0], ca)) - jobs.append(("crossover", FILES[1], cb)) - - paths = [] - for i, (_, _, child) in enumerate(jobs): - p = scratch / f"child_{i:03d}.dom" - dom.dump(child, str(p)) - paths.append(p) - scores = oracle.score_batch(paths, URB) # raises if any child is invalid - - sigs = {f: leaf_signature(root) for f, root in roots.items()} - stats: dict[str, list[tuple[float, float]]] = {} - for (name, f, child), s in zip(jobs, scores): - df = abs(s.fitness - parents[f].fitness) / parents[f].fitness - dg = geometry_perturbation(sigs[f], child) - stats.setdefault(name, []).append((df, dg)) - - print(f"{'operator':14s} {'n':>3s} {'mean |dF|/F':>12s} {'mean geom-pert':>15s}") - for name in sorted(stats): - dfs, dgs = zip(*stats[name]) - print(f"{name:14s} {len(dfs):3d} {np.mean(dfs):12.3f} {np.mean(dgs):15.3f}") - print(f"\nall {len(jobs)} children scored by the oracle without error") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/optimize_fullfitness.py b/experiments/optimize_fullfitness.py deleted file mode 100644 index 5105e1e..0000000 --- a/experiments/optimize_fullfitness.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Full-fitness frozen-topology optimisation. - -Drive the equal-offset division ratios with a derivative-free optimiser against -the REAL oracle fitness (the whole objective, not an area proxy) on a fixed -topology. This removes both confounds of the earlier sweeps (partial objective, -proxy target) and answers the only question that matters next: - - Is there geometry headroom above the EA's designs, or are they already - geometry-optima (=> the bottleneck is topology + the 0.5^n cliff)? - -For each candidate: report fitness/fails before and after. -""" - -import sys -from pathlib import Path - -import numpy as np -from scipy.optimize import minimize - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) - -from homemaker_layout import dom, oracle, solver # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples/programme-house" -SCRATCH = Path(__file__).resolve().parents[1] / "scratch" -_EPS = 0.02 - -CANDIDATES = [ - "candidate-002.dom", # ~0.0074 (MCP-refined) - "c964435454c459f86c3ed9a5a7621132.dom", # ~0.0037 (MCP baseline) -] - - -def optimise(src: Path, maxfev: int = 200): - import shutil - - shutil.copy(src, SCRATCH / "orig.dom") - s0 = oracle.score(SCRATCH / "orig.dom", URB) - - root = dom.load(str(src)) - free = solver.free_branches(root) - x0 = np.array([b.division[0] for b in free], dtype=float) - opt_path = SCRATCH / "opt.dom" - - best = {"f": s0.fitness, "fails": s0.n_fails, "x": x0.copy()} - - def neg_fitness(x: np.ndarray) -> float: - xc = np.clip(x, _EPS, 1 - _EPS) - for j, b in enumerate(free): - b.division = [float(xc[j]), float(xc[j])] - dom.dump(root, str(opt_path)) - try: - s = oracle.score(opt_path, URB) - except Exception: # noqa: BLE001 - return 1e9 - if s.fitness > best["f"]: - best.update(f=s.fitness, fails=s.n_fails, x=xc.copy()) - return -s.fitness - - minimize( - neg_fitness, x0, method="Nelder-Mead", - options={"maxfev": maxfev, "xatol": 1e-3, "fatol": 1e-12}, - ) - return s0, best, len(free) - - -def main() -> None: - SCRATCH.mkdir(exist_ok=True) - import shutil - - shutil.copy(EX / "patterns.config", SCRATCH / "patterns.config") - - maxfev = int(sys.argv[1]) if len(sys.argv) > 1 else 200 - for name in CANDIDATES: - s0, best, ndof = optimise(EX / name, maxfev) - ratio = best["f"] / s0.fitness if s0.fitness else float("inf") - verdict = "IMPROVED" if best["f"] > s0.fitness * 1.001 else "no gain" - print( - f"{name:42s} dof={ndof:2d} " - f"orig={s0.fitness:.5g}(fails {s0.n_fails}) " - f"opt={best['f']:.5g}(fails {best['fails']}) " - f"x{ratio:.2f} {verdict}", - flush=True, - ) - - -if __name__ == "__main__": - main() diff --git a/experiments/rebaseline_no_occlusion.py b/experiments/rebaseline_no_occlusion.py deleted file mode 100644 index 4029f82..0000000 --- a/experiments/rebaseline_no_occlusion.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Re-baseline the corpus under URB_NO_OCCLUSION=1 (homemaker-py-gp2). - -Occlusion/daylight is disabled in Urb behind the URB_NO_OCCLUSION env flag -(daylight -> 1 everywhere, CIEsky illumination factor pinned to 1, i.e. -simple crinkliness). Flipping the flag changes every score, so this script -records the new baseline in one pass: - - - per-file flag-on vs flag-off score and failure-set deltas (35 corpus files) - - batched oracle throughput under the flag - -Run the inner-loop reference gains separately: - URB_NO_OCCLUSION=1 python3 experiments/accept_innerloop.py 400 cma -""" - -from __future__ import annotations - -import math -import os -import shutil -import sys -import tempfile -import time -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -CORPUS = URB / "examples" / "programme-house" - - -def score_corpus(paths: list[Path], flag: bool) -> tuple[list[oracle.Score], float]: - os.environ.pop("URB_NO_OCCLUSION", None) - if flag: - os.environ["URB_NO_OCCLUSION"] = "1" - t0 = time.perf_counter() - scores = oracle.score_batch(paths, URB) - return scores, time.perf_counter() - t0 - - -def main() -> int: - with tempfile.TemporaryDirectory(prefix="rebaseline_") as tmp: - scratch = Path(tmp) - shutil.copy(CORPUS / "patterns.config", scratch) - paths = [Path(shutil.copy(d, scratch)) for d in sorted(CORPUS.glob("*.dom"))] - - off, t_off = score_corpus(paths, flag=False) - on, t_on = score_corpus(paths, flag=True) - - n_changed_score = n_changed_fails = 0 - print(f"{'file':42s} {'flag-off':>12s} {'flag-on':>12s} {'ratio':>7s} fails off->on") - for p, a, b in zip(paths, off, on): - score_changed = not math.isclose(a.fitness, b.fitness, rel_tol=1e-9) - fails_changed = a.fail_lines != b.fail_lines - n_changed_score += score_changed - n_changed_fails += fails_changed - ratio = b.fitness / a.fitness if a.fitness else float("inf") - mark = "*" if fails_changed else " " - print(f"{p.name:42s} {a.fitness:12.6g} {b.fitness:12.6g} {ratio:7.3f} " - f"{a.n_fails}->{b.n_fails}{mark}") - - n = len(paths) - print(f"\nscore changed: {n_changed_score}/{n} failure set changed: {n_changed_fails}/{n}") - print(f"batched s/dom: flag-off {t_off / n:.3f}, flag-on {t_on / n:.3f} " - f"(x{t_off / t_on:.2f})") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/refine_sweep.py b/experiments/refine_sweep.py deleted file mode 100644 index 0ed8b32..0000000 --- a/experiments/refine_sweep.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Population sweep: warm-start refine every evolved candidate and tally results. - -For each real .dom in the example dir, score it, run the solver as a geometry -optimiser (warm-start, no strip), and re-score. Reports how often bottom-up -sizing improves vs regresses total fitness, plus aggregate fail-count change. - -This is a breadth check on the solver-as-optimiser role; raw fitness is still -confounded by the 0.5^n failure cliff and any topological defects, so the -fail-count and per-candidate detail matter as much as the win/loss tally. -""" - -import shutil -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) - -from homemaker_layout import dom, oracle, programme, solver # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples/programme-house" - - -def _is_candidate(p: Path) -> bool: - # real designs: 32-hex hashes or candidate-NNN; skip init and our scratch - name = p.stem - return name not in {"init", "original", "roundtrip", "solved", "refined"} - - -def main() -> None: - scratch = Path(__file__).resolve().parents[1] / "scratch" - scratch.mkdir(exist_ok=True) - shutil.copy(EX / "patterns.config", scratch / "patterns.config") - targets = programme.load_programme(str(EX / "patterns.config")) - - doms = sorted(p for p in EX.glob("*.dom") if _is_candidate(p)) - win = loss = tie = 0 - fails_before = fails_after = 0 - rows = [] - for src in doms: - try: - shutil.copy(src, scratch / "orig.dom") - s0 = oracle.score(scratch / "orig.dom", URB) - root = dom.load(str(src)) - # gentlest refiner: nudge cut POSITIONS for programme-room area only, - # keep evolved cut angles and leave circulation/shape untouched. - solver.solve_ratios( - root, targets, strip=False, perpendicular=False, - weight_width=0.0, weight_proportion=0.0, min_width_generic=0.0, - ) - dom.dump(root, str(scratch / "ref.dom")) - s1 = oracle.score(scratch / "ref.dom", URB) - except Exception as e: # noqa: BLE001 - rows.append(f" {src.name:40s} ERROR {e}") - continue - fails_before += s0.n_fails - fails_after += s1.n_fails - if s1.fitness > s0.fitness * 1.001: - win += 1 - mark = "＋" - elif s1.fitness < s0.fitness * 0.999: - loss += 1 - mark = "－" - else: - tie += 1 - mark = "=" - rows.append( - f" {mark} {src.name:40s} {s0.fitness:.4g} -> {s1.fitness:.4g}" - f" fails {s0.n_fails}->{s1.n_fails}" - ) - - print("\n".join(rows)) - n = win + loss + tie - print(f"\n{n} candidates: {win} improved, {loss} regressed, {tie} tied") - print(f"total fails: {fails_before} -> {fails_after}") - - -if __name__ == "__main__": - main() diff --git a/experiments/resolve_ratios.py b/experiments/resolve_ratios.py deleted file mode 100644 index 27cc797..0000000 --- a/experiments/resolve_ratios.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Go/no-go experiment for bottom-up sizing. - -Take a real evolved .dom, throw away its division ratios, and re-solve them from -the programme's target sizes alone. Score three versions through the Perl oracle: - - original -- the evolved .dom as-is (baseline) - roundtrip -- loaded and re-emitted unmodified (checks dump fidelity) - solved -- ratios stripped to 0.5 then solved from programme targets - -If `solved` scores >= `original`, sizing can be recovered from the programme -without the evolved geometry, and the EA only needs to search topology. - -Usage: - python experiments/resolve_ratios.py [source.dom] [--urb /path/to/urb] -""" - -import argparse -import shutil -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) - -from homemaker_layout import dom, oracle, programme, solver # noqa: E402 - -DEFAULT_URB = Path("/home/bruno/src/urb") -DEFAULT_SRC = DEFAULT_URB / "examples/programme-house/candidate-002.dom" - - -def _score_in(scratch: Path, name: str, root: dom.Node, urb: Path) -> oracle.Score: - path = scratch / name - dom.dump(root, str(path)) - return oracle.score(path, urb) - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("source", nargs="?", default=str(DEFAULT_SRC)) - ap.add_argument("--urb", default=str(DEFAULT_URB)) - args = ap.parse_args() - - src = Path(args.source).resolve() - urb = Path(args.urb).resolve() - config = src.parent / "patterns.config" - - scratch = Path(__file__).resolve().parents[1] / "scratch" - scratch.mkdir(exist_ok=True) - shutil.copy(config, scratch / "patterns.config") - - targets = programme.load_programme(str(config)) - - # baseline: score the original file as-is - orig_copy = scratch / "original.dom" - shutil.copy(src, orig_copy) - s_orig = oracle.score(orig_copy, urb) - - # roundtrip: load + re-emit unmodified - s_round = _score_in(scratch, "roundtrip.dom", dom.load(str(src)), urb) - - # solved: strip ratios and re-solve from programme targets - root = dom.load(str(src)) - print("--- programme leaves BEFORE solve (ratios intact) ---") - print(solver.area_report(root, targets)) - res = solver.solve_ratios(root, targets, strip=True) - print("\n--- programme leaves AFTER solve (from targets, ratios stripped) ---") - print(solver.area_report(root, targets)) - print(f"\nsolver: cost={res.cost:.4f} nfev={res.nfev} success={res.success}") - s_solved = _score_in(scratch, "solved.dom", root, urb) - - # refined: warm-start from the evolved ratios (solver as geometry optimiser) - root2 = dom.load(str(src)) - solver.solve_ratios(root2, targets, strip=False) - s_refined = _score_in(scratch, "refined.dom", root2, urb) - - print("\n=== FITNESS (via urb-fitness.pl oracle) ===") - for label, s in ( - ("original", s_orig), - ("roundtrip", s_round), - ("solved", s_solved), - ("refined", s_refined), - ): - print(f" {label:9s} fitness={s.fitness:.10g} fails={s.n_fails}") - - print("\nVERDICT:") - print(f" solved (strip, frozen topology): {'>=' if s_solved.fitness >= s_orig.fitness else '<'} original") - print(f" refined (warm-start optimiser): {'>=' if s_refined.fitness >= s_orig.fitness else '<'} original") - - -if __name__ == "__main__": - main() diff --git a/experiments/run_search.py b/experiments/run_search.py deleted file mode 100644 index 28051d3..0000000 --- a/experiments/run_search.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -"""End-to-end memetic search on programme-house (homemaker-py-b39 acceptance). - -Runs driver.search from a corpus seed within a stated oracle-evaluation -budget, logs every improvement, writes the best design to -scratch/search_best.dom and independently re-scores it through the oracle to -prove the output is a valid .dom whose recorded fitness is reproducible. - -The interesting bar: the seed's geometry-only optimum (inner loop alone, -DESIGN §4.7 reference: c964435 -> 0.00674). Anything above that is value -added by *topology* search. - -Run under the go-forward fitness: - URB_NO_OCCLUSION=1 python3 experiments/run_search.py [budget] [rng-seed] [seed.dom] [out.dom] -""" - -from __future__ import annotations - -import math -import shutil -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from homemaker_layout import dom, driver, oracle # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples" / "programme-house" -SEED_FILE = EX / "c964435454c459f86c3ed9a5a7621132.dom" -OUT = Path(__file__).resolve().parents[1] / "scratch" / "search_best.dom" - - -def main() -> int: - budget = int(sys.argv[1]) if len(sys.argv) > 1 else 2000 - rng_seed = int(sys.argv[2]) if len(sys.argv) > 2 else 0 - seed_file = Path(sys.argv[3]) if len(sys.argv) > 3 else SEED_FILE - out = Path(sys.argv[4]) if len(sys.argv) > 4 else OUT - print(f"seed={seed_file.name} budget={budget} oracle evals, rng seed {rng_seed}", - flush=True) - - r = driver.search(dom.load(str(seed_file)), EX, budget=budget, - urb_root=URB, seed=rng_seed, log=lambda m: print(m, flush=True)) - - print(f"\ndone: {r.n_evals} oracle evals across {r.n_topologies} topologies") - print(f"best: {r.best.fitness:.6g} (fails {r.best.n_fails}) via {r.best.lineage}") - print("population: " + ", ".join(f"{p.fitness:.4g}/{p.n_fails}f" for p in r.population)) - - out.parent.mkdir(parents=True, exist_ok=True) - config_src = EX / "patterns.config" - if config_src.exists() and not (out.parent / "patterns.config").exists(): - shutil.copy(config_src, out.parent) - dom.dump(r.best.root, str(out)) - s = oracle.score(out, URB) - ok = math.isclose(s.fitness, r.best.fitness, rel_tol=1e-9) - print(f"\n{out} re-scored standalone: {s.fitness:.6g} ({s.n_fails} fails) " - f"-> {'MATCHES search record' if ok else 'MISMATCH'}") - return 0 if ok else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/experiments/run_search_scaled.py b/experiments/run_search_scaled.py index d517faa..ba66b55 100644 --- a/experiments/run_search_scaled.py +++ b/experiments/run_search_scaled.py @@ -109,7 +109,6 @@ def main() -> int: restart_patience=restart_patience, seed_adjacency_aware=adj, seed_proportion_aware=prop, - # urb_root not needed: use_native=True is the default ) elapsed = time.perf_counter() - t0 diff --git a/experiments/sweep_failtypes.py b/experiments/sweep_failtypes.py deleted file mode 100644 index 83ecd35..0000000 --- a/experiments/sweep_failtypes.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Equal-offset (perpendicular) warm-start sweep with failure-type histogram. - -Isolates which fitness constraints break when we move equal-offset cuts to fix -programme room sizes. Equal offset (a == b) keeps walls perpendicular on -near-rectangular plots, so any regression here is NOT a perpendicularity -artifact -- it tells us what sizing genuinely trades against. -""" - -import collections -import shutil -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) - -from homemaker_layout import dom, oracle, programme, solver # noqa: E402 - -URB = Path("/home/bruno/src/urb") -EX = URB / "examples/programme-house" - - -def classify(line: str) -> str: - s = line.strip().lower() - if not s or s == "---" or s.startswith(("- ", "type:", "issue:", "actual:", "min:")): - # structured YAML fails: pull the keyword if present - if "staircase" in s: - return "staircase" - if "access" in s or "inaccessible" in s: - return "access" - if not s or s == "---": - return "" - for key in ("perpendicular", "crinkliness", "width", "proportion", "size", - "adjacent", "access", "inaccessible", "level", "staircase"): - if key in s: - return "adjacency" if key == "adjacent" else ( - "access" if key == "inaccessible" else key) - return "other" - - -def tally(fails: str, counter: collections.Counter) -> None: - for line in fails.splitlines(): - t = classify(line) - if t: - counter[t] += 1 - - -def main() -> None: - scratch = Path(__file__).resolve().parents[1] / "scratch" - scratch.mkdir(exist_ok=True) - shutil.copy(EX / "patterns.config", scratch / "patterns.config") - targets = programme.load_programme(str(EX / "patterns.config")) - - before = collections.Counter() - after = collections.Counter() - doms = sorted(p for p in EX.glob("*.dom") if p.stem != "init") - - for src in doms: - shutil.copy(src, scratch / "orig.dom") - s0 = oracle.score(scratch / "orig.dom", URB) - root = dom.load(str(src)) - solver.solve_ratios( # equal offset, shape-aware (strong width/proportion) - root, targets, strip=False, perpendicular=True, - weight_width=3.0, weight_proportion=2.0, min_width_generic=1.5, - ) - dom.dump(root, str(scratch / "ref.dom")) - s1 = oracle.score(scratch / "ref.dom", URB) - tally(s0.fails, before) - tally(s1.fails, after) - - types = sorted(set(before) | set(after), key=lambda t: -(after[t] - before[t])) - print(f"{'failure type':16s} {'before':>8s} {'after':>8s} {'delta':>8s}") - for t in types: - print(f"{t:16s} {before[t]:8d} {after[t]:8d} {after[t] - before[t]:+8d}") - print(f"{'TOTAL':16s} {sum(before.values()):8d} {sum(after.values()):8d} " - f"{sum(after.values()) - sum(before.values()):+8d}") - - -if __name__ == "__main__": - main() diff --git a/src/homemaker_layout/driver.py b/src/homemaker_layout/driver.py index 98dd6b6..4e2d616 100644 --- a/src/homemaker_layout/driver.py +++ b/src/homemaker_layout/driver.py @@ -8,7 +8,8 @@ mandatory, homemaker-py-8cs: cold starts never catch up at equal budget). Budgets are stated and accounted in **oracle evaluations** (scored .dom files), never generations (§4.6 arithmetic). This driver is deliberately -small-scale for the Phase-2 proof on the batched Perl oracle; scaling up +small-scale for the Phase-2 proof on the batched Perl oracle (since removed, +DESIGN.md §39.21); scaling up waits for the native fitness (Phase 3). Cold-start bootstrap (homemaker-py-0px): when the seed is an undivided bare @@ -164,7 +165,7 @@ def random_topology(seed_root: dom.Node, n_leaves: int, return root -def _evaluate(root: dom.Node, programme_dir, urb_root, x0, budget, inner_kw, +def _evaluate(root: dom.Node, programme_dir, x0, budget, inner_kw, lineage: str, want_grade: bool = False, feasibility_max_shape_fails: int | None = None, best_n_fails: int | None = None, @@ -239,7 +240,7 @@ def _evaluate(root: dom.Node, programme_dir, urb_root, x0, budget, inner_kw, sig=genome.signature(root), n_hard=0, n_soft=pred) return ind, 1 r = innerloop.optimise(root, programme_dir, x0=x0, budget=budget, - urb_root=urb_root, conf_overrides=overrides, **inner_kw) + conf_overrides=overrides, **inner_kw) # §11.4: read the graded proximity scalar off the optimised tree. The inner # loop left ``root`` at the optimum (Lamarckian write-back), so re-scoring a # copy reproduces r.fitness/r.n_fails exactly and adds the grade. One extra @@ -276,7 +277,6 @@ def search( seed: int = 0, types: list[str] | None = None, inner_kw: dict | None = None, - urb_root=None, log=None, n_workers: int = 1, use_lex: bool = True, @@ -429,9 +429,6 @@ def search( exactly instead of un-dividing and regrowing it. Gated the same way as ``ruin_recreate`` (zero mutation weight unless enabled). """ - from .oracle import DEFAULT_URB_ROOT - - urb_root = urb_root or DEFAULT_URB_ROOT rng = np.random.default_rng(seed) inner_kw = dict(_CHILD_INNER_KW, **(inner_kw or {})) # §12.3 M3 reassociate (homemaker-py-9gp.2) is default-OFF: force its weight to @@ -607,7 +604,7 @@ def search( mx = feasibility_max_shape_fails if (filter_on and feasibility_filter) else None best_nf = result.best.n_fails if result.best is not None else None full = [ - (root, programme_dir, urb_root, x0, budget_, kw_, lin, use_grade, + (root, programme_dir, x0, budget_, kw_, lin, use_grade, mx, best_nf, leaf_sharing, superpose, max_share, conn_grade, collapse_insearch, multi_use, shapecurve_warmstart, shapecurve_prune) for root, x0, budget_, kw_, lin in tasks @@ -687,7 +684,7 @@ def search( # random divide+retype walk that leaves required rooms absent. _run_batch([_make_seed_task(str(i)) for i in range(pop_size)]) else: - seed_ind, used = _evaluate(copy.deepcopy(seed_root), programme_dir, urb_root, + seed_ind, used = _evaluate(copy.deepcopy(seed_root), programme_dir, x0=None, budget=seed_budget, inner_kw={}, lineage="seed", want_grade=use_grade, @@ -843,7 +840,7 @@ def polish_finish( # No polish: re-optimise the unfolded genome's ratios once and score it # canonically so the written .dom and reported fitness are honest. ind, used = _evaluate( - unfolded, programme_dir, None, x0=None, budget=rescore_budget, + unfolded, programme_dir, x0=None, budget=rescore_budget, inner_kw={}, lineage="unfold", leaf_sharing=False, superpose=superpose, multi_use=multi_use, collapse_insearch=collapse_insearch) r2 = SearchResult(best=ind, population=[ind], n_evals=used, n_topologies=1) diff --git a/src/homemaker_layout/fitness.py b/src/homemaker_layout/fitness.py index 8e0b14a..38e8b4f 100644 --- a/src/homemaker_layout/fitness.py +++ b/src/homemaker_layout/fitness.py @@ -12,7 +12,8 @@ Source of truth: ``Urb::Dom::Fitness::{Base,Leaf,Storey,ProgrammeDriven}``. DESCOPE (DESIGN.md §6, decision 2026-06-12): this ports *simple* crinkliness — the CIEsky illumination factor is pinned to 1, exactly what Urb computes under ``URB_NO_OCCLUSION=1``. ``quality_daylight`` is likewise pinned to 1. Parity -targets the *flagged* oracle, never stock Urb. +targeted the *flagged* oracle, never stock Urb. The oracle itself is gone +(DESIGN.md §39.21); this is now the only evaluator. Call ``dom.merge_divided(root)`` and rebuild graphs before ``process_storey`` — storey processing runs on the MERGED tree (two-phase pattern, see graph.py). diff --git a/src/homemaker_layout/innerloop.py b/src/homemaker_layout/innerloop.py index ef8ba02..bcf0616 100644 --- a/src/homemaker_layout/innerloop.py +++ b/src/homemaker_layout/innerloop.py @@ -7,8 +7,8 @@ ownership) against the FULL fitness. Never a proxy objective — §4.2 falsified that; the full objective's ``0.5^n`` failure cliff is what protects the inner loop from trading into new failures (§4.5). -Fitness defaults to the native Python evaluator (Phase 3). The Perl oracle -(``OracleEvaluator``) is kept for validation but is no longer used in search. +Fitness is the native Python evaluator. The Perl oracle it was ported from is +gone (DESIGN.md §39.21) -- "oracle call" below means one batched evaluation. Warm-starting from a parent's optimised ratios is ``x0=`` (§5 decision 6, Lamarckian inheritance). """ @@ -22,7 +22,7 @@ from pathlib import Path import numpy as np -from . import dom, oracle, solver +from . import dom, solver _EPS = 0.02 # keep cuts off the edges; matches solver/_experiments convention @@ -60,69 +60,9 @@ class Result: n_oracle_calls: int # perl invocations -class OracleEvaluator: - """Scores ratio vectors for a frozen topology via the batched oracle. - - Owns a scratch directory seeded with the programme config (and occlusion - field, if any) so ``urb-fitness.pl`` finds them in its working directory. - Use as a context manager, or call ``close()``. - """ - - _CONFIGS = ("patterns.config", "costs.config", "occlusion.field") - - def __init__( - self, - root: dom.Node, - programme_dir: str | Path, - urb_root: str | Path = oracle.DEFAULT_URB_ROOT, - ): - self.root = root - self.free = solver.free_branches(root) - self.urb_root = Path(urb_root) - self._dir = Path(tempfile.mkdtemp(prefix="innerloop_")) - for name in self._CONFIGS: - src = Path(programme_dir) / name - if src.exists(): - shutil.copy(src, self._dir) - self.n_evals = 0 - self.n_oracle_calls = 0 - - def __enter__(self) -> "OracleEvaluator": - return self - - def __exit__(self, *exc) -> None: - self.close() - - def close(self) -> None: - shutil.rmtree(self._dir, ignore_errors=True) - - @property - def x_current(self) -> np.ndarray: - # Midpoint projection: legacy designs carry slightly unequal offsets - # (a != b); (a+b)/2 is the least-damaging equal-offset start. - return np.array([(b.division[0] + b.division[1]) / 2 for b in self.free], dtype=float) - - def apply(self, x: np.ndarray) -> None: - xc = np.clip(x, _EPS, 1 - _EPS) - for j, b in enumerate(self.free): - b.division = [float(xc[j]), float(xc[j])] - - def evaluate(self, xs: list[np.ndarray]) -> list[oracle.Score]: - """Score a population of ratio vectors in one oracle invocation.""" - paths = [] - for i, x in enumerate(xs): - self.apply(x) - p = self._dir / f"member_{i:04d}.dom" - dom.dump(self.root, str(p)) - paths.append(p) - scores = oracle.score_batch(paths, self.urb_root) - self.n_evals += len(xs) - self.n_oracle_calls += 1 - return scores - def compass_search( - ev: OracleEvaluator, + ev: NativeEvaluator, x0: np.ndarray, budget: int = 200, step0: float = 0.25, @@ -196,7 +136,7 @@ def compass_search( def cma_search( - ev: OracleEvaluator, + ev: NativeEvaluator, x0: np.ndarray, budget: int = 200, sigmas: tuple[float, ...] = (0.05, 0.15), @@ -208,7 +148,7 @@ def cma_search( Covariance adaptation handles the diagonal ridges of the ``0.5^n`` landscape that stall axis-aligned pattern search; the ask/tell population - maps one-to-one onto ``OracleEvaluator.evaluate``. + maps one-to-one onto ``NativeEvaluator.evaluate``. One sigma does not fit all warm starts (measured at budget 200): 2f45907 needs a *local* phase — at sigma 0.15 the search wanders out of @@ -267,7 +207,7 @@ class _BudgetExhausted(Exception): def nm_search( - ev: "OracleEvaluator | NativeEvaluator", + ev: "NativeEvaluator", x0: np.ndarray, budget: int = 200, seed: int = 0, @@ -327,7 +267,7 @@ from dataclasses import dataclass as _dc @_dc class _NativeScore: - """oracle.Score-compatible result from native fitness.""" + """Scalar + failure set for one ratio vector.""" fitness: float fail_lines: tuple @@ -340,8 +280,7 @@ class _NativeScore: class NativeEvaluator: """Scores ratio vectors for a frozen topology via the native Python fitness. - Drop-in replacement for ``OracleEvaluator``; no temp directory, no Perl - startup overhead. Each ``evaluate`` call runs ``Fitness.score_with_fails`` + Each ``evaluate`` call runs ``Fitness.score_with_fails`` serially over the batch (all in-process, no parallelism needed at this scale). """ @@ -355,7 +294,7 @@ class NativeEvaluator: conf, cost = fit_mod.load_config(programme_dir, overrides=conf_overrides) self._fit = fit_mod.Fitness(conf, cost) self.n_evals = 0 - self.n_oracle_calls = 0 # kept for interface parity with OracleEvaluator + self.n_oracle_calls = 0 # legacy name: batches, not Perl calls def __enter__(self) -> "NativeEvaluator": return self @@ -376,7 +315,7 @@ class NativeEvaluator: def evaluate(self, xs: list[np.ndarray]) -> "list[_NativeScore]": """Score a batch of ratio vectors; returns objects with .fitness / - .n_fails / .fail_lines matching the oracle.Score interface.""" + .n_fails / .fail_lines.""" import copy results = [] @@ -396,8 +335,7 @@ def optimise( x0: np.ndarray | None = None, budget: int = 200, method: str = "nm", - use_native: bool = True, - urb_root: str | Path = oracle.DEFAULT_URB_ROOT, + conf_overrides: dict | None = None, **search_kw, ) -> Result: @@ -407,12 +345,10 @@ def optimise( parent's optimised ratios for a Lamarckian warm start. On return ``root`` carries the best ratios found. - ``use_native=True`` (default) uses the native Python fitness; set False to - fall back to the Perl oracle (kept for validation only). + evaluate with the native Python fitness. """ - ev_cls = NativeEvaluator if use_native else OracleEvaluator - ev_args = ((root, programme_dir, conf_overrides) if use_native - else (root, programme_dir, urb_root)) + ev_cls = NativeEvaluator + ev_args = (root, programme_dir, conf_overrides) with ev_cls(*ev_args) as ev: if x0 is None: x0 = ev.x_current diff --git a/src/homemaker_layout/oracle.py b/src/homemaker_layout/oracle.py deleted file mode 100644 index fa7d91e..0000000 --- a/src/homemaker_layout/oracle.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Phase-1 fitness oracle: score ``.dom`` files via Urb's ``urb-fitness.pl``. - -This is the only throwaway component. It shells out to the Perl evaluator so we -can validate the Python search core against the trusted fitness before porting -fitness to Python (Phase 3). ``urb-fitness.pl`` reads ``patterns.config`` from -its working directory, so the ``.dom`` must live beside the programme config. - -``urb-fitness.pl`` accepts many ``.dom`` paths per invocation; ``score_batch`` -exploits this so the ~0.65 s Perl startup amortises across a generation -(DESIGN.md §4.6: ~0.99 s/dom batched vs ~1.65 s/dom single). Note the Perl -script computes the occlusion field from the *first* dom in a batch and reuses -it for the rest; ``experiments/bench_batch_oracle.py`` verifies this leaves -corpus scores identical to single-file calls. - -Two flavours of Urb-side nondeterminism to know about (both from Perl's -per-process hash-order randomisation, neither a batching artifact): ``.fails`` -line *order* varies between runs (use ``Score.fail_lines``), and the score -itself can flip by ~1 ULP. Compare fitness with a relative tolerance -(``math.isclose(..., rel_tol=1e-12)``), never ``==``. -""" - -from __future__ import annotations - -import os -import subprocess -from dataclasses import dataclass -from pathlib import Path -from typing import Sequence - -import yaml - -DEFAULT_URB_ROOT = Path("/home/bruno/src/urb") - - -def _structured_fail_to_str(f: dict) -> str: - """Convert a structured failure dict (llm-agent-mcp branch format) to the - plain-text string that master urb/ProgrammeDriven.pm would have emitted.""" - t = f.get("type", "") - if t == "level": - return f"{f['space']} on wrong level (level {f['actual']}, expected {f['required']})" - if t == "missing": - code = f["code"] - c = f.get("constraint") - if c == "size": - return f"missing {code}: would need size check" - if c == "width": - return f"missing {code}: would need width check" - if c == "proportion": - return f"missing {code}: would need proportion check" - if c == "adjacency": - return f"missing {code}: would need adjacency to {f.get('target', '')}" - if c == "level": - return f"missing {code}: would need to be on level {f['required']}" - if c == "vertical": - return f"missing {code}: would need connection to {f.get('target', '')} below" - if f.get("critical"): - return f"missing required space: {code} (critical)" - return f"missing required space: {code}" - if t == "count": - return f"too many spaces: {f['code']} (found {f['actual']}, expected {f['expected']})" - if t == "adjacency": - return f"{f['node']} ({f['space']}) not adjacent to {f['target']}" - if t == "vertical": - return f"{f['space']} not connected to {f['target']} below" - if t == "staircase": - issue = f.get("issue", "") - if issue == "volume": - return "staircase volume" - if issue == "count": - actual = f.get("actual", 0) - if "min" in f: - return f"too few stairs ({actual}, min {f['min']})" - if "max" in f: - return f"too many stairs ({actual}, max {f['max']})" - if t == "storey": - if f.get("issue") == "limit": - return "storey limit" - if f.get("issue") == "minimum": - return "storey minimum" - if t == "access" and f.get("issue") == "no_outside_public_access": - return "no outside public access" - return str(f) - - -def _parse_fails(text: str) -> list[str]: - """Parse a .fails file that may contain a YAML block followed by plain-text - lines (urb branch format) or only plain-text lines (master format).""" - text = text.strip() - if not text: - return [] - if not text.startswith("---"): - return [line.strip() for line in text.splitlines() if line.strip()] - - # Split YAML block from trailing plain-text lines: YAML list items start - # with "- " or are indented; once we hit a non-indented, non-dash line - # that isn't blank or the "---" marker, the YAML part has ended. - yaml_lines: list[str] = [] - plain_lines: list[str] = [] - in_yaml = True - for line in text.splitlines(): - if in_yaml: - stripped = line.strip() - if not stripped or stripped == "---" or stripped.startswith("-") or line[:1] == " ": - yaml_lines.append(line) - else: - in_yaml = False - plain_lines.append(stripped) - else: - if line.strip(): - plain_lines.append(line.strip()) - - result: list[str] = [] - try: - doc = yaml.safe_load("\n".join(yaml_lines)) - if isinstance(doc, list): - for item in doc: - if isinstance(item, dict): - result.append(_structured_fail_to_str(item)) - except yaml.YAMLError: - pass - result.extend(plain_lines) - return result - - -@dataclass -class Score: - fitness: float - fails: str # raw .fails content (YAML and/or plain lines) - - @property - def fail_lines(self) -> tuple[str, ...]: - """Failure messages as a sorted tuple — Perl's per-process hash-order - randomisation shuffles the raw ``.fails`` line order between runs, so - comparisons must be order-insensitive.""" - return tuple(sorted(_parse_fails(self.fails))) - - @property - def n_fails(self) -> int: - return len(self.fail_lines) - - -def score_batch( - dom_paths: Sequence[str | Path], urb_root: str | Path = DEFAULT_URB_ROOT -) -> list[Score]: - """Score many ``.dom`` files in one ``urb-fitness.pl`` invocation. - - All files must live in the same directory (the working directory, where - ``patterns.config`` is found). Results are returned in input order. - """ - paths = [Path(p).resolve() for p in dom_paths] - if not paths: - return [] - cwd = paths[0].parent - for p in paths: - if p.parent != cwd: - raise ValueError(f"batch spans directories: {p} not in {cwd}") - Path(f"{p}.score").unlink(missing_ok=True) - Path(f"{p}.fails").unlink(missing_ok=True) - - urb_root = Path(urb_root).resolve() - env = {**os.environ, "DEBUG": "1", "URB_NO_OCCLUSION": "1"} - proc = subprocess.run( - ["perl", f"-I{urb_root}/lib", str(urb_root / "bin" / "urb-fitness.pl")] - + [p.name for p in paths], - cwd=cwd, - env=env, - capture_output=True, - text=True, - ) - - results = [] - for p in paths: - score_file = Path(f"{p}.score") - if not score_file.exists(): - raise RuntimeError( - f"urb-fitness.pl produced no score for {p}\n" - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" - ) - fails_file = Path(f"{p}.fails") - results.append( - Score( - fitness=float(score_file.read_text().strip()), - fails=fails_file.read_text() if fails_file.exists() else "", - ) - ) - return results - - -def score(dom_path: str | Path, urb_root: str | Path = DEFAULT_URB_ROOT) -> Score: - return score_batch([dom_path], urb_root)[0] diff --git a/tests/test_dom_corpus.py b/tests/test_dom_corpus.py index a905d93..44b79c4 100644 --- a/tests/test_dom_corpus.py +++ b/tests/test_dom_corpus.py @@ -1,12 +1,10 @@ """Corpus-backed tests for dom round-trip, free-branch ownership, and fitness parity. Skipped when the Urb checkout is absent (these need only its .dom files, not -perl). The parity tests compare native Python fitness against cached oracle -scores and failure sets (generated with URB_NO_OCCLUSION=1). +perl). Parity against the Perl oracle used to live here; the oracle is gone +(DESIGN.md §39.21) and those tests never ran, so they went with it. """ -import math -import re from pathlib import Path import pytest @@ -15,17 +13,6 @@ from homemaker_layout import dom, solver CORPUS = Path(__file__).parent.parent / "examples" / "programme-house" -# The parity fixtures are the Perl corpus, whose files Urb names by MD5. Any -# other .dom in this directory is something a later session generated -- a -# search artefact, a candidate, a seed -- and must NEVER be used as a parity -# fixture: its .score, if one exists, was written by the NATIVE scorer, so -# comparing against it compares the native scorer with itself and passes -# whatever the native scorer says. That is exactly what was happening here -# until 39.20; see `homemaker-py-*` for restoring real oracle fixtures. -_ORACLE_NAME = re.compile(r"^[0-9a-f]{32}\.dom$") -ORACLE_FIXTURES = sorted(p for p in CORPUS.glob("*.dom") - if _ORACLE_NAME.match(p.name)) - pytestmark = pytest.mark.skipif(not CORPUS.is_dir(), reason="Corpus not available") @@ -133,64 +120,7 @@ def _native_evaluate(src: Path): return score, frozenset(failures) -def _oracle_result(src: Path): - """Read cached oracle score and failure set (URB_NO_OCCLUSION=1).""" - from homemaker_layout.oracle import Score - - score_file = Path(str(src) + ".score") - fails_file = Path(str(src) + ".fails") - if not score_file.exists(): - pytest.skip( - f"No oracle score committed for {src.name}. `.gitignore` excludes " - "*.dom.score and *.dom.fails, and none has ever been tracked, so " - "native-vs-Perl parity is UNVERIFIED in this repository -- these " - "cases have always skipped on a clean checkout (DESIGN.md §39.20). " - "Regenerating them with the native scorer would not fix it; the " - "cache has to come from the Perl oracle.") - oracle_score = float(score_file.read_text().strip()) - oracle_fails = Score( - fitness=oracle_score, - fails=fails_file.read_text() if fails_file.exists() else "", - ).fail_lines - return oracle_score, frozenset(oracle_fails) -@pytest.mark.parametrize("src", ORACLE_FIXTURES, ids=lambda p: p.name) -def test_native_fitness_score_parity(src): - """Native score matches oracle within 1e-4 relative tolerance.""" - native_score, _ = _native_evaluate(src) - oracle_score, _ = _oracle_result(src) - assert math.isclose(native_score, oracle_score, rel_tol=1e-4, abs_tol=1e-15), ( - f"{src.name}: native={native_score:.6e} oracle={oracle_score:.6e}" - ) -@pytest.mark.parametrize("src", ORACLE_FIXTURES, ids=lambda p: p.name) -def test_native_fitness_fail_set_parity(src): - """Native failure set matches oracle failure set exactly.""" - _, native_fails = _native_evaluate(src) - _, oracle_fails = _oracle_result(src) - only_native = native_fails - oracle_fails - only_oracle = oracle_fails - native_fails - assert not only_native and not only_oracle, ( - f"{src.name}: only_native={sorted(only_native)} only_oracle={sorted(only_oracle)}" - ) - - -def test_parity_fixtures_are_never_session_artefacts(): - """Guard for the defect §39.20 records. - - The parity tests read a cached `.score` beside each `.dom` and treat it as - the Perl oracle's answer. Nothing in the file says who wrote it, so a `.dom` - left behind by a search run -- with a `.score` written by the NATIVE scorer - -- silently becomes a "parity" case that compares the native scorer with - itself. Three such cases were live and passing until the §39.19 objective - change made the native scorer disagree with its own stale output. - """ - for src in ORACLE_FIXTURES: - assert _ORACLE_NAME.match(src.name), src.name - stray = [p.name for p in CORPUS.glob("*.dom") - if not _ORACLE_NAME.match(p.name) - and Path(str(p) + ".score").exists()] - assert not set(stray) & {p.name for p in ORACLE_FIXTURES}, ( - f"session artefacts leaked into the parity fixtures: {stray}") diff --git a/tests/test_driver.py b/tests/test_driver.py index a56df28..255e244 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -28,7 +28,7 @@ def fake_inner(monkeypatch): observable.""" calls = [] - def fake_optimise(root, programme_dir, x0=None, budget=200, urb_root=None, **kw): + def fake_optimise(root, programme_dir, x0=None, budget=200, **kw): n_leaves = sum(len(lvl.leaves()) for lvl in dom.levels(root)) fitness = 1.0 / (1.0 + abs(12 - n_leaves)) + 1e-6 * len(calls) calls.append({"budget": budget, "x0": x0, "kw": kw}) @@ -166,7 +166,7 @@ def test_restart_keeps_elite_and_counts(monkeypatch): """§11.5: a stagnation restart fires, is counted, and preserves the best.""" # Saturating fake (no monotone tiebreaker, unlike `fake_inner`): fitness # peaks at 12 leaves and plateaus, so the best stalls and restarts trigger. - def fake_optimise(root, programme_dir, x0=None, budget=200, urb_root=None, **kw): + def fake_optimise(root, programme_dir, x0=None, budget=200, **kw): n_leaves = sum(len(lvl.leaves()) for lvl in dom.levels(root)) fitness = 1.0 / (1.0 + abs(12 - n_leaves)) return innerloop.Result( @@ -289,7 +289,7 @@ def test_shapecurve_warmstart_seeds_ratios_when_eligible(monkeypatch): divisions_at_optimise = [] - def fake_optimise(root, programme_dir, x0=None, budget=200, urb_root=None, **kw): + def fake_optimise(root, programme_dir, x0=None, budget=200, **kw): divisions_at_optimise.append( [tuple(b.division) for _, b in innerloop.free_with_keys(root)]) n_leaves = sum(len(lvl.leaves()) for lvl in dom.levels(root)) @@ -340,7 +340,7 @@ def test_shapecurve_warmstart_handles_multistorey(monkeypatch): it exactly as it would a single-storey one.""" from homemaker_layout import shapecurve - def fake_optimise(root, programme_dir, x0=None, budget=200, urb_root=None, **kw): + def fake_optimise(root, programme_dir, x0=None, budget=200, **kw): for _, b in innerloop.free_with_keys(root): b.division = [0.25, 0.25] return innerloop.Result( @@ -364,7 +364,7 @@ def test_shapecurve_warmstart_handles_multistorey(monkeypatch): HARBOR_L0 = Path(__file__).parent.parent / "examples" / "harbor-house-l0" -def _fake_optimise_ok(root, programme_dir, x0=None, budget=200, urb_root=None, **kw): +def _fake_optimise_ok(root, programme_dir, x0=None, budget=200, **kw): for _, b in innerloop.free_with_keys(root): b.division = [0.25, 0.25] return innerloop.Result( @@ -436,7 +436,7 @@ def test_shapecurve_prune_vetoes_heuristic_when_dp_feasible(monkeypatch): pytest.skip("harbor-house-l0 not available") root = dom.load(str(HARBOR_L0 / "init.dom")) ind, used = driver._evaluate( - root, HARBOR_L0, None, x0=None, budget=100, inner_kw={}, lineage="child", + root, HARBOR_L0, x0=None, budget=100, inner_kw={}, lineage="child", feasibility_max_shape_fails=0, best_n_fails=5, leaf_sharing=False, shapecurve_prune=True) @@ -461,7 +461,7 @@ def test_shapecurve_prune_hard_prunes_when_dp_infeasible_and_incumbent_perfect(m pytest.skip("harbor-house-l0 not available") root = dom.load(str(HARBOR_L0 / "init.dom")) ind, used = driver._evaluate( - root, HARBOR_L0, None, x0=None, budget=100, inner_kw={}, lineage="child", + root, HARBOR_L0, x0=None, budget=100, inner_kw={}, lineage="child", feasibility_max_shape_fails=0, best_n_fails=0, leaf_sharing=False, shapecurve_prune=True) @@ -485,7 +485,7 @@ def test_shapecurve_prune_defers_to_heuristic_when_incumbent_nonzero(monkeypatch pytest.skip("harbor-house-l0 not available") root = dom.load(str(HARBOR_L0 / "init.dom")) ind, used = driver._evaluate( - root, HARBOR_L0, None, x0=None, budget=100, inner_kw={}, lineage="child", + root, HARBOR_L0, x0=None, budget=100, inner_kw={}, lineage="child", feasibility_max_shape_fails=0, best_n_fails=5, leaf_sharing=False, shapecurve_prune=True) @@ -670,7 +670,7 @@ def test_use_tiers_prefers_fewer_hard_over_fewer_total_fails(monkeypatch): seed_root = dom.load(str(SEED_FILE)) calls = [] # first call is always the seed eval; every later call is a child - def fake_optimise(root, programme_dir, x0=None, budget=200, urb_root=None, **kw): + def fake_optimise(root, programme_dir, x0=None, budget=200, **kw): for _, b in innerloop.free_with_keys(root): b.division = [0.25, 0.25] is_seed = len(calls) == 0 diff --git a/tests/test_innerloop.py b/tests/test_innerloop.py index 96fb324..f23ceb2 100644 --- a/tests/test_innerloop.py +++ b/tests/test_innerloop.py @@ -1,10 +1,27 @@ """Inner-loop search tests against a fake evaluator (no perl, no oracle).""" +from dataclasses import dataclass + import numpy as np import pytest from homemaker_layout import innerloop -from homemaker_layout.oracle import Score + + +@dataclass +class _Score: + """Minimal evaluator result: what the optimisers actually read. + + Was `oracle.Score`; the Perl oracle is gone (DESIGN.md §39.21) and these + tests only ever needed a fitness with an empty failure set. + """ + + fitness: float + fail_lines: tuple = () + + @property + def n_fails(self) -> int: + return len(self.fail_lines) class FakeEvaluator: @@ -18,7 +35,7 @@ class FakeEvaluator: def evaluate(self, xs): self.n_evals += len(xs) self.n_oracle_calls += 1 - return [Score(fitness=self.fn(np.asarray(x)), fails="") for x in xs] + return [_Score(self.fn(np.asarray(x))) for x in xs] def concave(x): diff --git a/tests/test_oracle.py b/tests/test_oracle.py deleted file mode 100644 index 453506d..0000000 --- a/tests/test_oracle.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Oracle unit tests that do not invoke perl.""" - -import pytest - -from homemaker_layout import oracle - - -def test_fail_lines_sorted_and_filtered(): - s = oracle.Score(fitness=0.5, fails="---\nb fail\n\na fail\n \n") - assert s.fail_lines == ("a fail", "b fail") - assert s.n_fails == 2 - - -def test_fail_lines_empty(): - s = oracle.Score(fitness=0.5, fails="") - assert s.fail_lines == () - assert s.n_fails == 0 - - -def test_score_batch_empty_list(): - assert oracle.score_batch([]) == [] - - -def test_score_batch_rejects_cross_directory(tmp_path): - a = tmp_path / "a" / "x.dom" - b = tmp_path / "b" / "y.dom" - for p in (a, b): - p.parent.mkdir() - p.write_text("") - with pytest.raises(ValueError, match="batch spans directories"): - oracle.score_batch([a, b])