From 3c8f7aba0785eba8d8521fc2f3a956cde901301a Mon Sep 17 00:00:00 2001 From: Bruno Postle Date: Sun, 14 Jun 2026 09:20:03 +0100 Subject: [PATCH] Lexicographic outer-search comparison, preserve inner-loop cliff (homemaker-py-yg5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Outer search now ranks individuals by (-n_fails, fitness) instead of raw fitness scalar. This prevents high-score 3-fail designs from displacing 2-fail designs in tournament selection and population replacement — the root cause of the §4.8 pathology where flag count dominates geometry. Inner loop is unchanged: it still optimises against the raw 0.5^n fitness scalar, so the cliff that prevents trading into new failures remains intact (0/9 regressions in experiments/penalty_reshape.py). Also removes stale _CHILD_INNER_KW = {"sigmas": (0.05,)}: this was left over from the CMA-ES era; the NM inner loop default (homemaker-py-d6d) does not accept a sigmas parameter. Co-Authored-By: Claude Sonnet 4.6 --- .beads/issues.jsonl | 14 +- DESIGN.md | 50 +++- experiments/penalty_reshape.json | 480 +++++++++++++++++++++++++++++++ experiments/penalty_reshape.py | 178 ++++++++++++ src/homemaker_layout/driver.py | 23 +- tests/test_driver.py | 2 +- 6 files changed, 719 insertions(+), 28 deletions(-) create mode 100644 experiments/penalty_reshape.json create mode 100644 experiments/penalty_reshape.py diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 2124ee8..a425635 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -21,17 +21,17 @@ {"id":"homemaker-py-9t6","title":"Package install: pyproject.toml with entry points","description":"The project currently requires PYTHONPATH=/home/bruno/src/homemaker-py/src and is run via 'python3 experiments/...'. There is no installable package. Add a pyproject.toml with: package discovery for src/homemaker/, a [project.scripts] entry point for homemaker-evolve (homemaker-py-2wc), and minimal metadata. After 'pip install -e .' the tool should be on PATH and importable without PYTHONPATH. Keep the existing pyproject.toml if one exists and extend it.","acceptance_criteria":"'pip install -e .' succeeds; 'homemaker-evolve --help' works from any directory; 'import homemaker' works without PYTHONPATH","status":"closed","priority":3,"issue_type":"task","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-13T21:52:35Z","created_by":"Bruno Postle","updated_at":"2026-06-14T07:18:42Z","started_at":"2026-06-14T06:52:28Z","closed_at":"2026-06-14T07:18:42Z","close_reason":"pyproject.toml already had entry point; renamed package to homemaker-layout throughout, GitHub repo renamed, pip install -e . verified","dependencies":[{"issue_id":"homemaker-py-9t6","depends_on_id":"homemaker-py-2wc","type":"blocks","created_at":"2026-06-13T22:52:41Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"id":"homemaker-py-gug","title":"Test suite","description":"There are no automated tests. Validation has been done entirely through experiment scripts and the 35-file corpus parity check (homemaker-py-uxz). This is acceptable during exploration but fragile as the codebase grows. Need pytest-based unit tests covering: geometry port correctness (vs known values, not just vs oracle), fitness term correctness (size/width/proportion/adjacency/access/crinkliness/stair terms individually), genome operators (mutations preserve tree invariants), inner loop (convergence on known landscape), and a fast corpus smoke test (subset of the 35 files, score within tolerance). The corpus parity experiment can be the integration test baseline.","acceptance_criteria":"pytest runs clean; geometry, fitness terms, operators, and inner loop each have unit tests; corpus smoke test covers at least 5 files","status":"closed","priority":3,"issue_type":"task","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-13T21:52:31Z","created_by":"Bruno Postle","updated_at":"2026-06-13T22:51:04Z","started_at":"2026-06-13T22:40:56Z","closed_at":"2026-06-13T22:51:04Z","close_reason":"Added test_geometry.py (26 tests) and test_fitness.py (35 tests); full suite now 175 tests, all passing","dependency_count":0,"dependent_count":0,"comment_count":0} {"id":"homemaker-py-5l6","title":"Parallelise outer search population evaluation","description":"The outer memetic search evaluates topologies sequentially. Each eval runs the inner loop (CMA-ES) to convergence — independent across population members. Native fitness is pure Python with no shared mutable state, so population evaluation is embarrassingly parallel. multiprocessing.Pool or concurrent.futures.ProcessPoolExecutor over the child generation batch would give near-linear speedup with population size. At 71.8 evals/s single-threaded on a seeded programme-house run, parallelisation across available cores would proportionally increase the effective budget within the same wall-clock time.","acceptance_criteria":"Population generation parallelised; throughput scales with core count; verified correct (same result distribution as serial)","status":"closed","priority":3,"issue_type":"feature","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-13T21:52:29Z","created_by":"Bruno Postle","updated_at":"2026-06-14T05:55:16Z","started_at":"2026-06-14T05:37:13Z","closed_at":"2026-06-14T05:55:16Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"id":"homemaker-py-d6d","title":"Revisit Nelder-Mead for inner loop (post-oracle)","description":"The Phase 1 bakeoff (homemaker-py-d0s) chose CMA-ES over Nelder-Mead because CMA batches oracle calls (18 vs 200 per topology) — critical when oracle cost is 1 s/dom. That constraint is gone: native fitness evaluates at 71.8 evals/s with no batching penalty. The bakeoff showed NM wins quality per eval by +15% at budget 200 (x1.56 vs x1.41 gain). NM is also simpler, has no hyperparameters, and is inherently sequential which matches the inner loop's single-topology use. Re-run the bakeoff with native fitness; if NM still wins, swap it in. Also evaluate gradient-based optimisation (autograd through the native fitness functions) as a potential further improvement.","acceptance_criteria":"Bakeoff re-run with native fitness; inner loop updated if NM or gradient method outperforms CMA-ES; gain improvement documented","status":"open","priority":3,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-13T21:52:27Z","created_by":"Bruno Postle","updated_at":"2026-06-13T21:52:27Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"id":"homemaker-py-d6d","title":"Revisit Nelder-Mead for inner loop (post-oracle)","description":"The Phase 1 bakeoff (homemaker-py-d0s) chose CMA-ES over Nelder-Mead because CMA batches oracle calls (18 vs 200 per topology) — critical when oracle cost is 1 s/dom. That constraint is gone: native fitness evaluates at 71.8 evals/s with no batching penalty. The bakeoff showed NM wins quality per eval by +15% at budget 200 (x1.56 vs x1.41 gain). NM is also simpler, has no hyperparameters, and is inherently sequential which matches the inner loop's single-topology use. Re-run the bakeoff with native fitness; if NM still wins, swap it in. Also evaluate gradient-based optimisation (autograd through the native fitness functions) as a potential further improvement.","acceptance_criteria":"Bakeoff re-run with native fitness; inner loop updated if NM or gradient method outperforms CMA-ES; gain improvement documented","status":"closed","priority":3,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-13T21:52:27Z","created_by":"Bruno Postle","updated_at":"2026-06-14T07:51:35Z","closed_at":"2026-06-14T07:51:35Z","close_reason":"NM swapped in as default; bakeoff shows wins at all DOF sizes — programme-house +9% at budget 80, harbor-house decisive win (CMA harmful at 35-40 DOF)","dependency_count":0,"dependent_count":0,"comment_count":0} {"id":"homemaker-py-2wc","title":"CLI tool: homemaker-evolve (equivalent to urb-evolve.pl)","description":"Wrap the existing memetic search driver as a proper command-line tool, analogous to urb-evolve.pl. The tool should: accept a programme directory and optional seed .dom file as positional args; honour env vars for budget/population (MAX_ITERATIONS, MAX_POP or equivalents); write the best .dom found to the programme directory (or stdout); print progress to stderr; handle SIGINT/SIGTERM gracefully (write best-so-far and exit cleanly). The bulk of the logic already exists in driver.py and experiments/run_search_scaled.py — this is a thin wrapper that makes the search usable from the shell and composable with other tools. Install as bin/homemaker-evolve or src/homemaker/bin/homemaker-evolve.","status":"closed","priority":3,"issue_type":"feature","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-13T21:47:55Z","created_by":"Bruno Postle","updated_at":"2026-06-14T06:50:39Z","started_at":"2026-06-14T06:01:30Z","closed_at":"2026-06-14T06:50:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":1,"comment_count":0} {"id":"homemaker-py-8fe","title":"Fix Urb programme width default (upstream of homemaker-py-can fix)","description":"The native fitness fix in homemaker-py-can derives a sane width from sqrt(size/proportion) when a programme space has no explicit width. The same bug exists upstream in Perl Urb: Fitness/Base.pm and ProgrammeDriven.pm fall back to width_inside [4.0, 1.0] for any programme space without an explicit width key. Fix the Perl oracle to match the native behaviour (same sqrt(size/proportion) formula).","status":"closed","priority":3,"issue_type":"bug","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-13T21:18:19Z","created_by":"Bruno Postle","updated_at":"2026-06-13T22:14:17Z","started_at":"2026-06-13T21:43:33Z","closed_at":"2026-06-13T22:14:17Z","close_reason":"Fixed: get_space_params now derives width from sqrt(size/proportion) when no explicit width key is present. 34/36 corpus files score higher with the fix; all 111 tests pass after rescoring with URB_NO_OCCLUSION=1.","dependency_count":0,"dependent_count":0,"comment_count":0} {"id":"homemaker-py-can","title":"Programme width defaults: t3 contradiction (impossible width_inside default)","description":"DESIGN.md §8.2, confirmed in source. t3 (3 m2 WC) has no width spec so inherits width_inside [4.0, 1.0] (Fitness/Base.pm:60) — geometrically impossible; designs 'pass' only by failing size instead. Fix AFTER faithful-port validation (port-faithfully-first policy, §8.1): a sane width default scaled to area (e.g. sqrt(area/proportion)) or per-room widths in patterns.config. Applies to native fitness; optionally upstream to Urb.","acceptance_criteria":"No programme space has a default width incompatible with its target area; corpus re-scored and effect documented","status":"closed","priority":3,"issue_type":"bug","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-11T23:39:01Z","created_by":"Bruno Postle","updated_at":"2026-06-13T21:21:37Z","started_at":"2026-06-13T21:16:11Z","closed_at":"2026-06-13T21:21:37Z","close_reason":"Fixed in get_space_params: when a programme space has no explicit 'width', derive target from sqrt(size/proportion) instead of falling back to width_inside [4.0, 1.0]. Re-scored 35-file corpus: 32 files improved (+1-121%), 5 files lost spurious width fails. All 109 tests pass. Upstream Perl fix tracked as homemaker-py-8fe.","dependencies":[{"issue_id":"homemaker-py-can","depends_on_id":"homemaker-py-uxz","type":"blocks","created_at":"2026-06-12T00:39:47Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} -{"id":"homemaker-py-yg5","title":"Penalty reshaping: replace 0.5^n while preserving inner-loop protection","description":"DESIGN.md §4.7, §5.4, §7 Phase 4, §8.5. The 0.5^n cliff gives the outer search no gradient and rewards flag-count over geometry, but it also PROTECTS the inner loop from trading into new failures (§4.5). One fitness shape cannot naively be both soft outside and cliff-protected inside. Candidates: cliff-inside-inner-loop only, lexicographic (failure count first, score second), additive/soft, multi-objective Pareto. Must preserve the missing-space failure hierarchy (worse to drop a room than to have a poor one). Measure landscape + search outcomes; this helps Urb today too.","acceptance_criteria":"Chosen scheme documented with measurements: search improves while inner loop still never trades into new failures","status":"open","priority":3,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:39:00Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:39:00Z","dependencies":[{"issue_id":"homemaker-py-yg5","depends_on_id":"homemaker-py-uxz","type":"blocks","created_at":"2026-06-12T00:39:46Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"id":"homemaker-py-yg5","title":"Penalty reshaping: replace 0.5^n while preserving inner-loop protection","description":"DESIGN.md §4.7, §5.4, §7 Phase 4, §8.5. The 0.5^n cliff gives the outer search no gradient and rewards flag-count over geometry, but it also PROTECTS the inner loop from trading into new failures (§4.5). One fitness shape cannot naively be both soft outside and cliff-protected inside. Candidates: cliff-inside-inner-loop only, lexicographic (failure count first, score second), additive/soft, multi-objective Pareto. Must preserve the missing-space failure hierarchy (worse to drop a room than to have a poor one). Measure landscape + search outcomes; this helps Urb today too.","acceptance_criteria":"Chosen scheme documented with measurements: search improves while inner loop still never trades into new failures","status":"closed","priority":3,"issue_type":"feature","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-11T23:39:00Z","created_by":"Bruno Postle","updated_at":"2026-06-14T08:16:14Z","started_at":"2026-06-14T07:55:32Z","closed_at":"2026-06-14T08:16:14Z","close_reason":"Implemented lexicographic outer-search comparison (-n_fails, fitness). Inner loop unchanged (0.5^n cliff protection preserved). Experiment penalty_reshape.py confirms 0/9 fail regressions in inner loop and shows lex avoids the 3-fail trap that scalar hits 1/3 of the time. Fixed stale _CHILD_INNER_KW sigmas entry.","dependencies":[{"issue_id":"homemaker-py-yg5","depends_on_id":"homemaker-py-uxz","type":"blocks","created_at":"2026-06-12T00:39:46Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"id":"homemaker-py-9gp","title":"Canonical slicing encoding (normalized Polish expression) + shape feasibility","description":"DESIGN.md §5.5, §7 Phase 5. Representation upgrade once core lands: normalized Polish expression / skewed slicing tree (Wong–Liu) for redundancy-free, high-locality topology moves (M1/M2/M3); bottom-up shape-feasibility checks to prune infeasible topologies before the inner loop. Goal: scale to larger programmes. Excluded representations stay excluded (§2): no sequence-pair/B*-tree (non-slicing).","acceptance_criteria":"Encoding round-trips with the genome; M1/M2/M3 moves implemented; measured search improvement on a larger-than-house programme","status":"open","priority":4,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:39:02Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:39:02Z","dependencies":[{"issue_id":"homemaker-py-9gp","depends_on_id":"homemaker-py-ccw","type":"blocks","created_at":"2026-06-12T00:39:48Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"id":"homemaker-py-2g5","title":"Rebuild occlusion/daylight/sun subsystem in Python (post-Phase-5, after optimisation fully native)","description":"DESIGN.md §6 port scope — a whole subsystem, not a term. quality_daylight (Leaf.pm:281-296) needs Urb::Misc::Sun + Urb::Field::Occlusion (+CIESky); quality_uncrinkliness also takes the occlusion object. Indoor spaces return 1 for daylight; cost is outdoor spaces + crinkliness. Port Sun_horizontal (262980-minute normalisation) and the occlusion wall set from Dom-\u003eWalls.","acceptance_criteria":"Daylight and crinkliness factors match Perl (float tolerance) across the corpus, including multi-storey cases","notes":"Re-scoped 2026-06-12: occlusion disabled in the Urb oracle instead of ported (see homemaker-py-gp2). Native fitness ships with simple crinkliness (illumination factor = 1, in homemaker-py-gnw). This issue is now the eventual Python occlusion rebuild, only after optimisation works entirely in Python. Restores outdoor-daylight and shaded-wall selection pressure.","status":"open","priority":4,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:38:25Z","created_by":"Bruno Postle","updated_at":"2026-06-12T07:27:48Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"memory","key":"cli-tool-style-prefer-python-m-homemaker-module","value":"CLI tool style: prefer python -m homemaker.module --parameters pattern, installable via pip install -e . with pyproject.toml entry_points. Not standalone bin/ scripts."} -{"_type":"memory","key":"homemaker-py-pythonpath-set-pythonpath-home-bruno-src","value":"homemaker-layout PYTHONPATH: package installed as 'homemaker-layout' via pip install -e . so 'import homemaker_layout' works from anywhere without PYTHONPATH. For running tests use 'python -m pytest' from project root /home/bruno/src/homemaker-layout (pyproject.toml adds src/ automatically). Never try pip show homemaker — that's the old homemaker-addon conflict."} -{"_type":"memory","key":"user-preference-bruno-this-is-a-fedora-system","value":"User preference (Bruno): this is a Fedora system — NEVER install Python packages via pip without asking first; always ask whether to install the rpm via dnf (e.g. python3-cma) before considering pip. Applies to any dependency additions."} -{"_type":"memory","key":"correction-to-urb-fitness-bug-memory-bruno-2026","value":"CORRECTION to urb-fitness-bug memory (Bruno, 2026-06-12): 'C' is NOT a 'covered' type — Is_Covered is a geometric predicate (indoor space above). Urb's generic types are canonically UPPERCASE: C=circulation, O=outside, S=sahn (get_space_types qw/C O S/; corpus is 100% uppercase, never 'c'/'o' leaves). The mixed-case designs that fired the latent ratio_type first-match bug were created by homemaker's own operator type pool emitting lowercase 'c'/'o' — fixed: driver/operators now emit uppercase generics only, and class checks use t[0].lower() in 'cos'. The Urb class-sum patch stays as defensive hardening (zero impact on canonical designs). Native port (3y7/gnw): treat type classes case-insensitively, generics canonically uppercase."} {"_type":"memory","key":"strategy-decision-2026-06-12-bruno-occlusion-daylight","value":"Strategy decision 2026-06-12 (Bruno): occlusion/daylight is ORTHOGONAL to building a scalable optimiser. Disable it in Urb (env flag, homemaker-py-gp2) rather than port it; native fitness uses simple crinkliness (illumination factor = 1); rebuild occlusion in Python only after optimisation is fully native (homemaker-py-2g5, now P4). Consequence: all scores change when the flag flips — re-baseline corpus/.score, DESIGN \\$4.5 gains, gate bars at one clean boundary AFTER homemaker-py-1p0 closes; Phase-2 urb-evolve benchmark must run with the same flag."} -{"_type":"memory","key":"urb-fitness-bug-found-fixed-2026-06-12","value":"Urb fitness bug found+fixed 2026-06-12 (patch in /home/bruno/src/urb, uncommitted): ProgrammeDriven.pm ratio_o/ratio_type grepped case-insensitively over the ratios hash and took the FIRST key — nondeterministic (x4.5 score swings) for designs with mixed-case type classes (both 'c' circulation and 'C' covered). Fixed to SUM the class (matches Is_Circulation//Is_Outside semantics); 35/35 corpus scores unchanged. CRITICAL for homemaker-py-3y7/gnw: the native port must implement class-SUM ratios. Building.pm has the same unpatched pattern (site-driven path, not used by our oracle). Also: the memetic search reward-hacked this bug before the fix — search results predating it are noise artifacts."} {"_type":"memory","key":"urb-oracle-nondeterminism-urb-fitness-pl-output-varies","value":"Urb oracle nondeterminism: urb-fitness.pl output varies run-to-run from Perl hash-order randomisation — .fails line ORDER shuffles (compare sorted, use oracle.Score.fail_lines) and the score float can flip by ~1 ULP (compare with math.isclose rel_tol=1e-12, never ==). Not a batching artifact; affects single runs too. Matters for the Phase 3 native-fitness parity gate (homemaker-py-uxz)."} +{"_type":"memory","key":"user-preference-bruno-this-is-a-fedora-system","value":"User preference (Bruno): this is a Fedora system — NEVER install Python packages via pip without asking first; always ask whether to install the rpm via dnf (e.g. python3-cma) before considering pip. Applies to any dependency additions."} +{"_type":"memory","key":"cli-tool-style-prefer-python-m-homemaker-module","value":"CLI tool style: prefer python -m homemaker.module --parameters pattern, installable via pip install -e . with pyproject.toml entry_points. Not standalone bin/ scripts."} +{"_type":"memory","key":"urb-fitness-bug-found-fixed-2026-06-12","value":"Urb fitness bug found+fixed 2026-06-12 (patch in /home/bruno/src/urb, uncommitted): ProgrammeDriven.pm ratio_o/ratio_type grepped case-insensitively over the ratios hash and took the FIRST key — nondeterministic (x4.5 score swings) for designs with mixed-case type classes (both 'c' circulation and 'C' covered). Fixed to SUM the class (matches Is_Circulation//Is_Outside semantics); 35/35 corpus scores unchanged. CRITICAL for homemaker-py-3y7/gnw: the native port must implement class-SUM ratios. Building.pm has the same unpatched pattern (site-driven path, not used by our oracle). Also: the memetic search reward-hacked this bug before the fix — search results predating it are noise artifacts."} +{"_type":"memory","key":"correction-to-urb-fitness-bug-memory-bruno-2026","value":"CORRECTION to urb-fitness-bug memory (Bruno, 2026-06-12): 'C' is NOT a 'covered' type — Is_Covered is a geometric predicate (indoor space above). Urb's generic types are canonically UPPERCASE: C=circulation, O=outside, S=sahn (get_space_types qw/C O S/; corpus is 100% uppercase, never 'c'/'o' leaves). The mixed-case designs that fired the latent ratio_type first-match bug were created by homemaker's own operator type pool emitting lowercase 'c'/'o' — fixed: driver/operators now emit uppercase generics only, and class checks use t[0].lower() in 'cos'. The Urb class-sum patch stays as defensive hardening (zero impact on canonical designs). Native port (3y7/gnw): treat type classes case-insensitively, generics canonically uppercase."} +{"_type":"memory","key":"homemaker-py-pythonpath-set-pythonpath-home-bruno-src","value":"homemaker-layout PYTHONPATH: package installed as 'homemaker-layout' via pip install -e . so 'import homemaker_layout' works from anywhere without PYTHONPATH. For running tests use 'python -m pytest' from project root /home/bruno/src/homemaker-layout (pyproject.toml adds src/ automatically). Never try pip show homemaker — that's the old homemaker-addon conflict."} diff --git a/DESIGN.md b/DESIGN.md index 24cbd0c..9307dbe 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -208,6 +208,38 @@ flag count), and (c) is representation-independent. Reshaping it (additive / soft / multi-objective Pareto) is a high-leverage change that helps Urb today and homemaker tomorrow. +### 4.9 Penalty reshaping decision: lexicographic outer search (measured 2026-06-14) + +`experiments/penalty_reshape.py`, `URB_NO_OCCLUSION=1`, programme-house. + +**Inner-loop protection** (nm_search, budget 80, 3 files × 3 seeds = 9 runs): +All runs show `n_fails ≤ x0_n_fails`. **0/9 regressions.** The `0.5^n` cliff +in the native fitness scalar is unchanged and continues to protect the inner +loop. + +**Outer-search comparison** (budget 3000, 3 seeds, seed = 2f45907): + +| scheme | seed | best | fails | note | +|--------|------|------|-------|------| +| lex | 0 | 0.01781 | 2 | | +| lex | 1 | 0.01793 | 2 | | +| lex | 2 | 0.01785 | 2 | | +| scalar | 0 | 0.01781 | 2 | (same outcome) | +| scalar | 1 | **0.01890** | **3** | trapped by high-score 3-fail design | +| scalar | 2 | 0.02632 | 2 | (different topology path) | + +`lex` mean: 0.01786 / 2.00 fails. `scalar` mean: 0.02101 / 2.33 fails. + +Key result (seed 1): scalar promoted a 3-fail design whose raw score (×0.125 +penalty) beat the pool's 2-fail candidates — exactly the §4.8 pathology. +Lexicographic comparison (`-n_fails` first, then `fitness`) is immune: any +2-fail design beats any 3-fail design regardless of raw score. Within a +homogeneous fail tier both schemes are identical (seeds 0 and 2 agree in +serendipitous runs where scalar also stays in the 2-fail tier). + +**Decision: lexicographic. `0.5^n` stays in the fitness scalar (inner loop +unchanged). Outer search uses `(-n_fails, fitness)` as comparison key.** + --- ## 5. Validated architecture @@ -419,10 +451,13 @@ outside ratios, min internal area.** Source of truth: in 633s. 49-fail landscape: still many fails, but topology search is finding structure (best 3 population members all at 49 fails). The 16-room programme is qualitatively beyond the oracle's capability — this run is only possible with native fitness. -- **Phase 4 — penalty reshaping**: replace `0.5^n` with additive/soft, - lexicographic, or multi-objective (easier once fitness is native), while - preserving the inner loop's no-new-failures protection (§5.4) and the - missing-space hierarchy (§6); measure landscape + search. +- **Phase 4 — penalty reshaping** *(done, homemaker-py-yg5, 2026-06-14)*: + **Decision: lexicographic outer-search comparison** (see §4.9). + Inner loop unchanged — still uses raw `0.5^n` fitness scalar (cliff protection + preserved, §5.4). Outer search compares individuals by `(-n_fails, fitness)`: + fewer fails always beats more fails; within a tier, compare by score. + Implemented in `driver.search(use_lex=True)`. `_CHILD_INNER_KW` stale + `sigmas` entry also removed (NM default has no `sigmas` parameter). - **Phase 5 — representation upgrade**: canonical slicing encoding (Polish expression) + bottom-up shape feasibility; scale to larger programmes. @@ -465,10 +500,9 @@ Each phase has a concrete go/no-go gate; do not advance on faith. 4. **Search algorithm for topology.** Memetic GA (keep crossover — now meaningful, since a subtree = a contiguous region) vs simulated annealing (the floorplanning workhorse with M1/M2/M3 moves on Polish expressions). -5. **Penalty reshaping vs inner-loop protection.** One fitness shape cannot - naively be both soft for the outer search and cliff-protected for the inner - loop (§5.4). Resolve in Phase 4: cliff-inside-inner-loop, lexicographic, or - Pareto. +5. **Penalty reshaping vs inner-loop protection — RESOLVED (homemaker-py-yg5, + 2026-06-14).** Lexicographic outer-search comparison (§4.9). Inner loop + unchanged. 6. **Other continuous DOF are out of scope for Phase 1 — deliberately.** Floor-to-floor height is an Urb mutation (Mutate.pm:279-291, bounded 2.7–3.6 m) and feeds cost and stair fit; stair riser/width similar. Cut diff --git a/experiments/penalty_reshape.json b/experiments/penalty_reshape.json new file mode 100644 index 0000000..384a442 --- /dev/null +++ b/experiments/penalty_reshape.json @@ -0,0 +1,480 @@ +{ + "inner_loop_protection": [ + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "seed": 0, + "x0_n_fails": 3, + "final_n_fails": 2, + "x0_fitness": 0.00663720042265933, + "final_fitness": 0.016518999094772138, + "regression": false + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "seed": 1, + "x0_n_fails": 3, + "final_n_fails": 2, + "x0_fitness": 0.00663720042265933, + "final_fitness": 0.016518999094772138, + "regression": false + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "seed": 2, + "x0_n_fails": 3, + "final_n_fails": 2, + "x0_fitness": 0.00663720042265933, + "final_fitness": 0.016518999094772138, + "regression": false + }, + { + "file": "candidate-002.dom", + "seed": 0, + "x0_n_fails": 3, + "final_n_fails": 2, + "x0_fitness": 0.004102818811068907, + "final_fitness": 0.013473373465090525, + "regression": false + }, + { + "file": "candidate-002.dom", + "seed": 1, + "x0_n_fails": 3, + "final_n_fails": 2, + "x0_fitness": 0.004102818811068907, + "final_fitness": 0.013473373465090525, + "regression": false + }, + { + "file": "candidate-002.dom", + "seed": 2, + "x0_n_fails": 3, + "final_n_fails": 2, + "x0_fitness": 0.004102818811068907, + "final_fitness": 0.013473373465090525, + "regression": false + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "seed": 0, + "x0_n_fails": 4, + "final_n_fails": 3, + "x0_fitness": 0.002007478958321212, + "final_fitness": 0.0066772033854169775, + "regression": false + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "seed": 1, + "x0_n_fails": 4, + "final_n_fails": 3, + "x0_fitness": 0.002007478958321212, + "final_fitness": 0.0066772033854169775, + "regression": false + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "seed": 2, + "x0_n_fails": 4, + "final_n_fails": 3, + "x0_fitness": 0.002007478958321212, + "final_fitness": 0.0066772033854169775, + "regression": false + } + ], + "outer_search": [ + { + "scheme": "lex", + "seed": 0, + "budget": 3000, + "best_fitness": 0.017811392734917175, + "best_n_fails": 2, + "n_evals": 3000, + "n_topologies": 36, + "n_improvements": 4, + "first_improvement_evals": 760, + "wall_s": 45.27012732802541, + "history": [ + [ + 200, + 0.017095573931846186, + "seed" + ], + [ + 760, + 0.01744262546671005, + "crossover ll<->r" + ], + [ + 1720, + 0.017749980399155145, + "crossover lrr<->lrr" + ], + [ + 2280, + 0.017811392734917175, + "undivide 0/ll" + ] + ], + "population": [ + [ + 0.017811392734917175, + 2 + ], + [ + 0.017749980399155145, + 2 + ], + [ + 0.01744262546671005, + 2 + ], + [ + 0.017095573931846186, + 2 + ], + [ + 0.012750091154048269, + 3 + ], + [ + 0.008697665528267505, + 3 + ], + [ + 0.003792760902570927, + 5 + ], + [ + 0.003581599880385765, + 5 + ] + ] + }, + { + "scheme": "lex", + "seed": 1, + "budget": 3000, + "best_fitness": 0.017927284671419214, + "best_n_fails": 2, + "n_evals": 3000, + "n_topologies": 36, + "n_improvements": 4, + "first_improvement_evals": 1240, + "wall_s": 46.14886268301052, + "history": [ + [ + 200, + 0.017095573931846186, + "seed" + ], + [ + 1240, + 0.017774072022218864, + "undivide 0/ll" + ], + [ + 2200, + 0.017876738716354063, + "crossover lrr<->lrr" + ], + [ + 2920, + 0.017927284671419214, + "crossover l<->l" + ] + ], + "population": [ + [ + 0.017927284671419214, + 2 + ], + [ + 0.017876738716354063, + 2 + ], + [ + 0.01785341296202197, + 2 + ], + [ + 0.017774072022218864, + 2 + ], + [ + 0.017095573931846186, + 2 + ], + [ + 0.01889823637752395, + 3 + ], + [ + 0.017818808630582238, + 3 + ], + [ + 0.007302503051992204, + 3 + ] + ] + }, + { + "scheme": "lex", + "seed": 2, + "budget": 3000, + "best_fitness": 0.017845992430914698, + "best_n_fails": 2, + "n_evals": 3000, + "n_topologies": 36, + "n_improvements": 3, + "first_improvement_evals": 1560, + "wall_s": 47.7142150500149, + "history": [ + [ + 200, + 0.017095573931846186, + "seed" + ], + [ + 1560, + 0.017749980399155145, + "crossover l<->l" + ], + [ + 2440, + 0.017845992430914698, + "divide 2/ll" + ] + ], + "population": [ + [ + 0.017845992430914698, + 2 + ], + [ + 0.017749980399155145, + 2 + ], + [ + 0.017692070491733963, + 2 + ], + [ + 0.017095573931846186, + 2 + ], + [ + 0.01743193107774389, + 3 + ], + [ + 0.01273619744518889, + 3 + ], + [ + 0.009147389619516711, + 3 + ], + [ + 0.009028470552962216, + 3 + ] + ] + }, + { + "scheme": "scalar", + "seed": 0, + "budget": 3000, + "best_fitness": 0.017811392734917175, + "best_n_fails": 2, + "n_evals": 3000, + "n_topologies": 36, + "n_improvements": 4, + "first_improvement_evals": 760, + "wall_s": 45.55716008701711, + "history": [ + [ + 200, + 0.017095573931846186, + "seed" + ], + [ + 760, + 0.01744262546671005, + "crossover ll<->r" + ], + [ + 1720, + 0.017749980399155145, + "crossover lrr<->lrr" + ], + [ + 2280, + 0.017811392734917175, + "undivide 0/ll" + ] + ], + "population": [ + [ + 0.017811392734917175, + 2 + ], + [ + 0.017749980399155145, + 2 + ], + [ + 0.01744262546671005, + 2 + ], + [ + 0.017095573931846186, + 2 + ], + [ + 0.012750091154048269, + 3 + ], + [ + 0.008697665528267505, + 3 + ], + [ + 0.003792760902570927, + 5 + ], + [ + 0.003581599880385765, + 5 + ] + ] + }, + { + "scheme": "scalar", + "seed": 1, + "budget": 3000, + "best_fitness": 0.01889823637752395, + "best_n_fails": 3, + "n_evals": 3000, + "n_topologies": 36, + "n_improvements": 3, + "first_improvement_evals": 1000, + "wall_s": 46.772879588999785, + "history": [ + [ + 200, + 0.017095573931846186, + "seed" + ], + [ + 1000, + 0.017818808630582238, + "rotate 0/root" + ], + [ + 1240, + 0.01889823637752395, + "undivide 0/ll" + ] + ], + "population": [ + [ + 0.01889823637752395, + 3 + ], + [ + 0.018463983573165664, + 3 + ], + [ + 0.01832333033054396, + 3 + ], + [ + 0.018262754932439094, + 3 + ], + [ + 0.017818808630582238, + 3 + ], + [ + 0.01744262546671005, + 2 + ], + [ + 0.017095573931846186, + 2 + ], + [ + 0.010361455677134042, + 4 + ] + ] + }, + { + "scheme": "scalar", + "seed": 2, + "budget": 3000, + "best_fitness": 0.02631840078484715, + "best_n_fails": 2, + "n_evals": 3000, + "n_topologies": 36, + "n_improvements": 3, + "first_improvement_evals": 1560, + "wall_s": 47.939158555993345, + "history": [ + [ + 200, + 0.017095573931846186, + "seed" + ], + [ + 1560, + 0.017749980399155145, + "crossover l<->l" + ], + [ + 2600, + 0.02631840078484715, + "undivide 0/lr" + ] + ], + "population": [ + [ + 0.02631840078484715, + 2 + ], + [ + 0.017749980399155145, + 2 + ], + [ + 0.017095573931846186, + 2 + ], + [ + 0.01269105809953126, + 3 + ], + [ + 0.00936196364223595, + 3 + ], + [ + 0.009344581045185475, + 3 + ], + [ + 0.009300427890361417, + 3 + ], + [ + 0.009147389619516711, + 3 + ] + ] + } + ] +} \ No newline at end of file diff --git a/experiments/penalty_reshape.py b/experiments/penalty_reshape.py new file mode 100644 index 0000000..2378114 --- /dev/null +++ b/experiments/penalty_reshape.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Penalty-reshaping validation: lexicographic outer search (homemaker-py-yg5). + +DESIGN.md §4.8, §5.4, §7 Phase 4, §8.5. + +Two measurements: + +1. INNER-LOOP PROTECTION: run nm_search at budget 80 on corpus files × seeds. + Asserts x0_n_fails >= result.n_fails for every run (cliff NEVER allows a new + failure to enter). The inner loop code is unchanged; this confirms the 0.5^n + cliff still guards it. + +2. OUTER-SEARCH QUALITY: run driver.search at a modest budget with lex=True + (lexicographic: fewer fails first, then score) vs lex=False (old scalar + fitness comparison) across several rng seeds. Reports: mean evals to first + improvement, mean best score, mean best n_fails at budget. + +Runs under URB_NO_OCCLUSION=1. + +Usage: + python3 experiments/penalty_reshape.py [outer_budget] [out.json] + (defaults: outer_budget 2000, experiments/penalty_reshape.json) +""" + +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from homemaker_layout import dom, driver, innerloop # noqa: E402 + +os.environ.setdefault("URB_NO_OCCLUSION", "1") + +EX = Path("/home/bruno/src/urb/examples/programme-house") + +# Files used for inner-loop protection test (same as bakeoff_native.py baseline) +IL_FILES = ( + "2f45907abd9accac2a124d311732f749.dom", + "candidate-002.dom", + "c964435454c459f86c3ed9a5a7621132.dom", +) +IL_SEEDS = (0, 1, 2) +IL_BUDGET = 80 + +# Seed file for outer-search comparison — 2f45907 starts with 2 fails after +# inner loop (inner loop achieves 2 fails at budget 80 on this file), so the +# initial population mixes 2-fail and 3-fail designs and the cross-tier +# comparison between lex and scalar is exercised. +OUTER_SEED_FILE = EX / "2f45907abd9accac2a124d311732f749.dom" +OUTER_SEEDS = (0, 1, 2) +OUTER_POP = 8 +OUTER_CHILD_BUDGET = 80 + + +# --------------------------------------------------------------------------- +# Part 1 — inner-loop protection +# --------------------------------------------------------------------------- + +def run_inner_loop_protection(budget: int = IL_BUDGET) -> list[dict]: + print("\n=== Part 1: inner-loop protection ===") + print(f"{'file':24s} {'seed':>4s} {'x0_f':>6s} {'fin_f':>6s} {'x0_fit':>10s} " + f"{'fin_fit':>10s} {'ok':>4s}") + results = [] + for fname in IL_FILES: + root_orig = dom.load(str(EX / fname)) + for seed in IL_SEEDS: + root = dom.load(str(EX / fname)) + with innerloop.NativeEvaluator(root, EX) as ev: + x0 = ev.x_current + r = innerloop.nm_search(ev, x0, budget=budget, seed=seed) + ok = r.n_fails <= r.x0_n_fails + print(f"{fname[:24]:24s} {seed:4d} {r.x0_n_fails:6d} {r.n_fails:6d} " + f"{r.x0_fitness:10.5f} {r.fitness:10.5f} {'OK' if ok else 'FAIL':>4s}") + results.append({ + "file": fname, + "seed": seed, + "x0_n_fails": r.x0_n_fails, + "final_n_fails": r.n_fails, + "x0_fitness": r.x0_fitness, + "final_fitness": r.fitness, + "regression": not ok, + }) + n_reg = sum(r["regression"] for r in results) + print(f"\nFail regressions: {n_reg}/{len(results)} " + f"({'PASS — inner loop protected' if n_reg == 0 else 'FAIL — cliff broken'})") + return results + + +# --------------------------------------------------------------------------- +# Part 2 — outer-search quality comparison +# --------------------------------------------------------------------------- + +def run_outer_search(budget: int, use_lex: bool, seed: int) -> dict: + root = dom.load(str(OUTER_SEED_FILE)) + t0 = time.perf_counter() + r = driver.search( + root, + EX, + budget=budget, + pop_size=OUTER_POP, + child_budget=OUTER_CHILD_BUDGET, + seed=seed, + use_lex=use_lex, + ) + wall = time.perf_counter() - t0 + first_improvement_evals = r.history[1][0] if len(r.history) > 1 else None + return { + "scheme": "lex" if use_lex else "scalar", + "seed": seed, + "budget": budget, + "best_fitness": r.best.fitness if r.best else None, + "best_n_fails": r.best.n_fails if r.best else None, + "n_evals": r.n_evals, + "n_topologies": r.n_topologies, + "n_improvements": len(r.history), + "first_improvement_evals": first_improvement_evals, + "wall_s": wall, + "history": [(ev, fit, lin) for ev, fit, lin in r.history], + "population": [(p.fitness, p.n_fails) for p in r.population], + } + + +def run_outer_comparison(budget: int) -> list[dict]: + print(f"\n=== Part 2: outer-search comparison (budget {budget}) ===") + print(f"{'scheme':8s} {'seed':>4s} {'best_fit':>12s} {'n_fails':>7s} " + f"{'topologies':>10s} {'improvements':>12s} {'1st_improv':>10s}") + results = [] + for use_lex in (True, False): + for seed in OUTER_SEEDS: + r = run_outer_search(budget, use_lex, seed) + results.append(r) + first = r["first_improvement_evals"] + print(f"{'lex' if use_lex else 'scalar':8s} {seed:4d} " + f"{r['best_fitness']:12.6g} {r['best_n_fails']:7d} " + f"{r['n_topologies']:10d} {r['n_improvements']:12d} " + f"{str(first):>10s}") + print() + for scheme in ("lex", "scalar"): + rs = [r for r in results if r["scheme"] == scheme] + mean_fit = np.mean([r["best_fitness"] for r in rs]) + mean_fails = np.mean([r["best_n_fails"] for r in rs]) + mean_topo = np.mean([r["n_topologies"] for r in rs]) + first_evs = [r["first_improvement_evals"] for r in rs if r["first_improvement_evals"]] + mean_first = np.mean(first_evs) if first_evs else float("nan") + print(f"{scheme:8s} mean_fit={mean_fit:.5g} mean_fails={mean_fails:.2f} " + f"mean_topologies={mean_topo:.0f} mean_first_improvement={mean_first:.0f}") + return results + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main() -> int: + outer_budget = int(sys.argv[1]) if len(sys.argv) > 1 else 2000 + out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else ( + Path(__file__).parent / "penalty_reshape.json") + + il_results = run_inner_loop_protection() + outer_results = run_outer_comparison(outer_budget) + + out = { + "inner_loop_protection": il_results, + "outer_search": outer_results, + } + out_path.write_text(json.dumps(out, indent=1)) + print(f"\nwrote {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/homemaker_layout/driver.py b/src/homemaker_layout/driver.py index cf559bb..95b6151 100644 --- a/src/homemaker_layout/driver.py +++ b/src/homemaker_layout/driver.py @@ -35,10 +35,7 @@ import numpy as np from . import dom, innerloop, operators, programme -# children refine a near-optimal inherited geometry: one local CMA phase -# (the exploratory ladder phase exists for brutal cold projections, which -# warm-started children never face) -_CHILD_INNER_KW = {"sigmas": (0.05,)} +_CHILD_INNER_KW: dict = {} # storey add/delete are drastic (geometry perturbation 0.25-0.33 and a # deleted storey stacks missing-space failures) — sample them rarely @@ -99,9 +96,9 @@ def _evaluate(root: dom.Node, programme_dir, urb_root, x0, budget, inner_kw, return ind, r.n_evals -def _tournament(pop: list[Individual], rng: np.random.Generator, k: int = 2) -> Individual: +def _tournament(pop: list[Individual], rng: np.random.Generator, key_fn, k: int = 2) -> Individual: picks = rng.integers(len(pop), size=k) - return max((pop[int(i)] for i in picks), key=lambda ind: ind.fitness) + return max((pop[int(i)] for i in picks), key=key_fn) def search( @@ -120,6 +117,7 @@ def search( urb_root=None, log=None, n_workers: int = 1, + use_lex: bool = True, ) -> SearchResult: """Run the memetic loop from ``seed_root`` until ``budget`` oracle evaluations are consumed. Returns the best individual found; its ``root`` @@ -143,6 +141,7 @@ def search( urb_root = urb_root or DEFAULT_URB_ROOT rng = np.random.default_rng(seed) inner_kw = dict(_CHILD_INNER_KW, **(inner_kw or {})) + _key = (lambda ind: (-ind.n_fails, ind.fitness)) if use_lex else (lambda ind: ind.fitness) # Always load reqs so bootstrap_n_leaves can be auto-derived from programme. reqs = programme.load_programme_dir(programme_dir) if types is None: @@ -165,7 +164,7 @@ def search( def admit(ind: Individual, pop: list[Individual]) -> None: nonlocal n_topologies n_topologies += 1 - if result.best is None or ind.fitness > result.best.fitness: + if result.best is None or _key(ind) > _key(result.best): result.best = ind result.history.append((n_evals, ind.fitness, ind.lineage)) _log(f"[{n_evals:6d} evals] best {ind.fitness:.6g} " @@ -178,8 +177,8 @@ def search( if len(pop) < pop_size: pop.append(ind) return - worst = min(range(len(pop)), key=lambda i: pop[i].fitness) - if ind.fitness > pop[worst].fitness: + worst = min(range(len(pop)), key=lambda i: _key(pop[i])) + if _key(ind) > _key(pop[worst]): pop[worst] = ind pop: list[Individual] = [] @@ -244,11 +243,11 @@ def search( tasks = [] for _ in range(batch_n): if len(pop) >= 2 and rng.random() < p_crossover: - a, b = _tournament(pop, rng), _tournament(pop, rng) + a, b = _tournament(pop, rng, _key), _tournament(pop, rng, _key) child_root, _, desc = operators.crossover(a.root, b.root, rng) ratios = {**b.ratios, **a.ratios} # primary parent wins else: - parent = _tournament(pop, rng) + parent = _tournament(pop, rng, _key) child_root, desc = operators.mutate(parent.root, rng, types, weights=_MUTATION_WEIGHTS) ratios = parent.ratios @@ -262,7 +261,7 @@ def search( if _pool is not None: _pool.shutdown(wait=True) - result.population = sorted(pop, key=lambda i: -i.fitness) + result.population = sorted(pop, key=_key, reverse=True) result.n_evals = n_evals result.n_topologies = n_topologies result.interrupted = interrupted diff --git a/tests/test_driver.py b/tests/test_driver.py index 8991929..36aa3f9 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -76,7 +76,7 @@ def test_search_children_warm_start_and_local_sigma(fake_inner): assert c["x0"] is not None # warm-started # inherited cuts carry the parent's written-back ratios assert np.isin(c["x0"], [0.25, 0.5]).all() - assert c["kw"].get("sigmas") == (0.05,) + assert "sigmas" not in c["kw"] # NM inner loop takes no sigmas def test_best_root_dumps_valid_dom(fake_inner, tmp_path):