From bc61f8cb7322d759288576517945a22512cd20f8 Mon Sep 17 00:00:00 2001 From: Bruno Postle Date: Sat, 13 Jun 2026 09:46:04 +0100 Subject: [PATCH] Bake-off: CMA-ES confirmed as inner-loop optimiser (homemaker-py-d0s) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 4-way comparison (NM / CMA-ES / compass / compass-ms) over 3 corpus files × 3 seeds at budget 200, cold-start, URB_NO_OCCLUSION=1. CMA-ES wins on batch-efficiency (18 oracle calls vs 200 for NM, 12x speedup on Perl startup amortisation per §4.6) with acceptable quality (x1.41 @200 vs NM's x1.56). Compass stalls on narrow-valley landscapes and introduces fail regressions. NM flagged as Phase 3+ candidate once native fitness removes oracle call overhead. Co-Authored-By: Claude Sonnet 4.6 --- DESIGN.md | 26 +- experiments/bakeoff_innerloop.json | 767 +++++++++++++++++++++++++++++ experiments/bakeoff_innerloop.py | 224 +++++++++ 3 files changed, 1012 insertions(+), 5 deletions(-) create mode 100644 experiments/bakeoff_innerloop.json create mode 100644 experiments/bakeoff_innerloop.py diff --git a/DESIGN.md b/DESIGN.md index 983b584..3aa73e7 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -374,11 +374,27 @@ Each phase has a concrete go/no-go gate; do not advance on faith. `width_inside` default (Fitness/Base.pm:60) — geometrically impossible; the original "passes" only by failing `size` instead. *Confirmed in source.* Need a sane width default scaled to area, or per-room widths. -3. **Inner-loop optimiser choice.** Nelder-Mead worked for diagnostics; DOF is - small (≈ rooms−1, 6–7 on the corpus), so CMA-ES may be overkill — batched - multi-start pattern search parallelises across the oracle and is simpler. - Resolve via the Phase 1 bake-off, not upfront. Gradient-based becomes an - option once native fitness is differentiable-ish. +3. **Inner-loop optimiser choice — RESOLVED (homemaker-py-d0s, 2026-06-13).** + Bake-off over 3 files × 4 methods × 3 seeds at budget 200 + (`experiments/bakeoff_innerloop.py`), cold-start, `URB_NO_OCCLUSION=1`: + + | method | x@40 | x@80 | x@200 | s/eval | oracle calls | fails+ | + |-------------|------|------|-------|--------|--------------|--------| + | Nelder-Mead | 1.45 | 1.50 | 1.56 | 2.05 | 200 | 0 | + | CMA-ES | 1.09 | 1.32 | 1.41 | 1.69 | 18 | 0 | + | compass | 0.71 | 0.92 | 1.48 | 1.69 | 12 | 3 | + | compass-ms | 0.71 | 0.92 | 0.92 | 1.44 | 13 | 4 | + + **Decision: keep CMA-ES (already the default) for the Perl oracle era.** + Nelder-Mead wins quality per eval (+x0.15 at @200) but is inherently + sequential — 200 Perl invocations vs 18 for CMA (§4.6 batching matters). + Compass stalls on narrow-valley landscapes (2f45907: x0.62 vs x1.30) and + introduces fail regressions 3/9 runs. Multi-start compass wastes budget + on phase splits. + + **Phase 3+ note:** once native fitness replaces the oracle, oracle-call count + disappears. Revisit Nelder-Mead then — its quality advantage is real. + Gradient-based (autograd through native fitness) is also an option. 4. **Search algorithm for topology.** Memetic GA (keep crossover — now meaningful, since a subtree = a contiguous region) vs simulated annealing (the floorplanning workhorse with M1/M2/M3 moves on Polish expressions). diff --git a/experiments/bakeoff_innerloop.json b/experiments/bakeoff_innerloop.json new file mode 100644 index 0000000..5ae17c0 --- /dev/null +++ b/experiments/bakeoff_innerloop.json @@ -0,0 +1,767 @@ +{ + "budget": 200, + "checkpoints": [ + 40, + 80, + 120, + 200 + ], + "runs": [ + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "nm", + "seed": 0, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.015336134791349723, + "80": 0.01625453592736873, + "120": 0.016813214603667533, + "200": 0.016970841073693777 + }, + "final_fitness": 0.016970841073693777, + "final_n_fails": 2, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 426.10284887799935 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "nm", + "seed": 1, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.015336134791349723, + "80": 0.01625453592736873, + "120": 0.016813214603667533, + "200": 0.016970841073693777 + }, + "final_fitness": 0.016970841073693777, + "final_n_fails": 2, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 407.97355182700267 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "nm", + "seed": 2, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.015336134791349723, + "80": 0.01625453592736873, + "120": 0.016813214603667533, + "200": 0.016970841073693777 + }, + "final_fitness": 0.016970841073693777, + "final_n_fails": 2, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 410.4154980900057 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "cma", + "seed": 0, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.013602853938370742, + "80": 0.016553033822579822, + "120": 0.016553033822579822, + "200": 0.016553033822579822 + }, + "final_fitness": 0.02127655034590672, + "final_n_fails": 2, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 350.45485657599056 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "cma", + "seed": 1, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.01095718684601756, + "80": 0.02048062509654076, + "120": 0.02048062509654076, + "200": 0.02048062509654076 + }, + "final_fitness": 0.02048062509654076, + "final_n_fails": 2, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 347.38566490099765 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "cma", + "seed": 2, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.011174318189865662, + "80": 0.011174318189865662, + "120": 0.018967095390790795, + "200": 0.018967095390790795 + }, + "final_fitness": 0.018967095390790795, + "final_n_fails": 2, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 337.9982590199943 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "compass", + "seed": 0, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.006607488139901861, + "80": 0.006607488139901861, + "120": 0.007209518879898406, + "200": 0.008009790075052892 + }, + "final_fitness": 0.008278786574187515, + "final_n_fails": 3, + "n_evals": 221, + "n_oracle_calls": 11, + "wall_s": 368.2416331849963 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "compass", + "seed": 1, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.006607488139901861, + "80": 0.006607488139901861, + "120": 0.007170981596472924, + "200": 0.008175449506286243 + }, + "final_fitness": 0.008399547907246088, + "final_n_fails": 3, + "n_evals": 221, + "n_oracle_calls": 11, + "wall_s": 380.4755447469943 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "compass", + "seed": 2, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.006607488139901861, + "80": 0.006607488139901861, + "120": 0.006912734615831356, + "200": 0.008078395888024203 + }, + "final_fitness": 0.008260349591825748, + "final_n_fails": 3, + "n_evals": 221, + "n_oracle_calls": 11, + "wall_s": 369.9571806669992 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "compass-ms", + "seed": 0, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.006607488139901861, + "80": 0.006607488139901861, + "120": 0.006607488139901861, + "200": 0.006607488139901861 + }, + "final_fitness": 0.006607488139901861, + "final_n_fails": 3, + "n_evals": 202, + "n_oracle_calls": 12, + "wall_s": 329.4596275669901 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "compass-ms", + "seed": 1, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.006607488139901861, + "80": 0.006607488139901861, + "120": 0.006607488139901861, + "200": 0.006607488139901861 + }, + "final_fitness": 0.006607488139901861, + "final_n_fails": 3, + "n_evals": 202, + "n_oracle_calls": 12, + "wall_s": 331.6751732940029 + }, + { + "file": "2f45907abd9accac2a124d311732f749.dom", + "method": "compass-ms", + "seed": 2, + "dof": 7, + "orig_fitness": 0.01303535257266566, + "orig_n_fails": 2, + "x0_fitness": 0.006460366122034485, + "x0_n_fails": 3, + "best_at": { + "40": 0.006607488139901861, + "80": 0.006607488139901861, + "120": 0.006607488139901861, + "200": 0.006607488139901861 + }, + "final_fitness": 0.006607488139901861, + "final_n_fails": 3, + "n_evals": 202, + "n_oracle_calls": 12, + "wall_s": 353.22266581900476 + }, + { + "file": "candidate-002.dom", + "method": "nm", + "seed": 0, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.013043378536981178, + "80": 0.013162194510538643, + "120": 0.013541863507298595, + "200": 0.013861258388999518 + }, + "final_fitness": 0.013861258388999518, + "final_n_fails": 2, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 533.6580855659995 + }, + { + "file": "candidate-002.dom", + "method": "nm", + "seed": 1, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.013043378536981178, + "80": 0.013162194510538643, + "120": 0.013541863507298595, + "200": 0.013861258388999518 + }, + "final_fitness": 0.013861258388999518, + "final_n_fails": 2, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 534.7732095019892 + }, + { + "file": "candidate-002.dom", + "method": "nm", + "seed": 2, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.013043378536981178, + "80": 0.013162194510538643, + "120": 0.013541863507298595, + "200": 0.013861258388999518 + }, + "final_fitness": 0.013861258388999518, + "final_n_fails": 2, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 529.9340490540053 + }, + { + "file": "candidate-002.dom", + "method": "cma", + "seed": 0, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.008766176333159788, + "80": 0.011349351963634282, + "120": 0.01183788578257949, + "200": 0.01183788578257949 + }, + "final_fitness": 0.01183788578257949, + "final_n_fails": 2, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 478.9317944670038 + }, + { + "file": "candidate-002.dom", + "method": "cma", + "seed": 1, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.011654770458049522, + "80": 0.011654770458049522, + "120": 0.011654770458049522, + "200": 0.011654770458049522 + }, + "final_fitness": 0.011654770458049522, + "final_n_fails": 2, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 486.4617367609899 + }, + { + "file": "candidate-002.dom", + "method": "cma", + "seed": 2, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.007280771403204401, + "80": 0.010704424796621367, + "120": 0.011790159199403088, + "200": 0.011790159199403088 + }, + "final_fitness": 0.011790159199403088, + "final_n_fails": 1, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 486.73567426699447 + }, + { + "file": "candidate-002.dom", + "method": "compass", + "seed": 0, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.004059651815091663, + "80": 0.01100524588060076, + "120": 0.013031039518511447, + "200": 0.014373318674033223 + }, + "final_fitness": 0.01473833833441545, + "final_n_fails": 2, + "n_evals": 211, + "n_oracle_calls": 12, + "wall_s": 502.3361668680009 + }, + { + "file": "candidate-002.dom", + "method": "compass", + "seed": 1, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.00921749875086076, + "80": 0.01007215847138918, + "120": 0.010784398878431061, + "200": 0.013581520956957 + }, + "final_fitness": 0.014220594074265345, + "final_n_fails": 2, + "n_evals": 213, + "n_oracle_calls": 12, + "wall_s": 482.41149059700547 + }, + { + "file": "candidate-002.dom", + "method": "compass", + "seed": 2, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.004120081557419885, + "80": 0.005109588901973775, + "120": 0.010453965023981411, + "200": 0.014012902439663506 + }, + "final_fitness": 0.014332429744919455, + "final_n_fails": 2, + "n_evals": 213, + "n_oracle_calls": 12, + "wall_s": 482.2521033469966 + }, + { + "file": "candidate-002.dom", + "method": "compass-ms", + "seed": 0, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.004059651815091663, + "80": 0.01100524588060076, + "120": 0.01100524588060076, + "200": 0.01100524588060076 + }, + "final_fitness": 0.01100524588060076, + "final_n_fails": 2, + "n_evals": 213, + "n_oracle_calls": 14, + "wall_s": 333.32965936099936 + }, + { + "file": "candidate-002.dom", + "method": "compass-ms", + "seed": 1, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.00921749875086076, + "80": 0.01007215847138918, + "120": 0.01007215847138918, + "200": 0.01007215847138918 + }, + "final_fitness": 0.01007215847138918, + "final_n_fails": 2, + "n_evals": 213, + "n_oracle_calls": 14, + "wall_s": 401.0064252880111 + }, + { + "file": "candidate-002.dom", + "method": "compass-ms", + "seed": 2, + "dof": 6, + "orig_fitness": 0.008075772316711586, + "orig_n_fails": 2, + "x0_fitness": 0.004059651815091663, + "x0_n_fails": 3, + "best_at": { + "40": 0.004120081557419885, + "80": 0.005109588901973775, + "120": 0.005109588901973775, + "200": 0.005109588901973775 + }, + "final_fitness": 0.005109588901973775, + "final_n_fails": 3, + "n_evals": 215, + "n_oracle_calls": 14, + "wall_s": 282.3956634509959 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "nm", + "seed": 0, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.006177486909077849, + "80": 0.006527788282834294, + "120": 0.0066662305157972855, + "200": 0.006693148459352215 + }, + "final_fitness": 0.006693148459352215, + "final_n_fails": 3, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 287.4831781459943 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "nm", + "seed": 1, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.006177486909077849, + "80": 0.006527788282834294, + "120": 0.0066662305157972855, + "200": 0.006693148459352215 + }, + "final_fitness": 0.006693148459352215, + "final_n_fails": 3, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 276.6431823440071 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "nm", + "seed": 2, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.006177486909077849, + "80": 0.006527788282834294, + "120": 0.0066662305157972855, + "200": 0.006693148459352213 + }, + "final_fitness": 0.006693148459352213, + "final_n_fails": 3, + "n_evals": 200, + "n_oracle_calls": 200, + "wall_s": 275.51684825500706 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "cma", + "seed": 0, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.00533036008277192, + "80": 0.00533036008277192, + "120": 0.00533036008277192, + "200": 0.00533036008277192 + }, + "final_fitness": 0.00533036008277192, + "final_n_fails": 3, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 225.46989931999997 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "cma", + "seed": 1, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.004648328603302138, + "80": 0.005836221739161555, + "120": 0.005836221739161555, + "200": 0.005836221739161555 + }, + "final_fitness": 0.005836221739161555, + "final_n_fails": 3, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 227.0608354230062 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "cma", + "seed": 2, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.004653776166165647, + "80": 0.004808176606999774, + "120": 0.004808176606999774, + "200": 0.004808176606999774 + }, + "final_fitness": 0.004808176606999774, + "final_n_fails": 3, + "n_evals": 208, + "n_oracle_calls": 18, + "wall_s": 226.25115464300325 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "compass", + "seed": 0, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.002909077000944207, + "80": 0.005636022639392401, + "120": 0.005637234383710328, + "200": 0.006368553115957026 + }, + "final_fitness": 0.006368553115957026, + "final_n_fails": 3, + "n_evals": 213, + "n_oracle_calls": 12, + "wall_s": 228.6235062899941 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "compass", + "seed": 1, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.0038443413369024806, + "80": 0.004505393404296923, + "120": 0.005335631011587571, + "200": 0.007639377813299466 + }, + "final_fitness": 0.0077190451924048515, + "final_n_fails": 2, + "n_evals": 213, + "n_oracle_calls": 12, + "wall_s": 230.63287285399565 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "compass", + "seed": 2, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.003989234037259822, + "80": 0.003993271664133415, + "120": 0.0083599248344224, + "200": 0.011177097393828111 + }, + "final_fitness": 0.011177097393828111, + "final_n_fails": 2, + "n_evals": 213, + "n_oracle_calls": 12, + "wall_s": 228.19498966900574 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "compass-ms", + "seed": 0, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.002909077000944207, + "80": 0.005636022639392401, + "120": 0.005636022639392401, + "200": 0.005636022639392401 + }, + "final_fitness": 0.005636022639392401, + "final_n_fails": 3, + "n_evals": 213, + "n_oracle_calls": 14, + "wall_s": 222.05841742099437 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "compass-ms", + "seed": 1, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.0038443413369024806, + "80": 0.004505393404296923, + "120": 0.004505393404296923, + "200": 0.004505393404296923 + }, + "final_fitness": 0.004505393404296923, + "final_n_fails": 2, + "n_evals": 215, + "n_oracle_calls": 14, + "wall_s": 228.25147005799226 + }, + { + "file": "c964435454c459f86c3ed9a5a7621132.dom", + "method": "compass-ms", + "seed": 2, + "dof": 6, + "orig_fitness": 0.0039977449092860615, + "orig_n_fails": 3, + "x0_fitness": 0.0019865661983037492, + "x0_n_fails": 4, + "best_at": { + "40": 0.003989234037259822, + "80": 0.003993271664133415, + "120": 0.003993271664133415, + "200": 0.003993271664133415 + }, + "final_fitness": 0.003993271664133415, + "final_n_fails": 3, + "n_evals": 215, + "n_oracle_calls": 14, + "wall_s": 223.43627467899933 + } + ] +} \ No newline at end of file diff --git a/experiments/bakeoff_innerloop.py b/experiments/bakeoff_innerloop.py new file mode 100644 index 0000000..a097b21 --- /dev/null +++ b/experiments/bakeoff_innerloop.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +"""Inner-loop optimiser bake-off at equal oracle budgets (homemaker-py-d0s). + +DESIGN.md §7 Phase 1(b), §8.3. DOF is only ≈ rooms−1 (6–7 on the corpus), so +the question is fitness gained per oracle evaluation, not asymptotic power. +Candidates: + + nm multi-start Nelder-Mead (scipy) — the §4.5 diagnostic optimiser. + Inherently sequential: ONE dom per oracle call, so the Perl + startup never amortises (§4.6). + cma multi-phase CMA-ES (innerloop.cma_search), one batched oracle + call per generation. + compass single-start batched compass search with pattern moves + random + augmentation (innerloop.compass_search). + compass-ms multi-start compass: budget split across restarts (x0 first, + then random starts), global best kept. + +Protocol: cold start from each file's equal-offset projection (x_current), +one run per (method, file, seed), best-so-far recorded after every oracle +call. Fitness-at-budget-B is read off the trace (evals ≤ B), so methods are +compared at exactly equal budgets regardless of batch granularity; checkpoint +budgets bracket the driver's real operating points (child_budget=80 warm, +seed_budget=200 cold). Wall-clock and oracle-invocation counts are recorded +per run. + +Runs under URB_NO_OCCLUSION=1 (set by this script — the gp2 re-baseline flag; +all benchmarks must use it). + +Usage: python3 experiments/bakeoff_innerloop.py [budget] [out.json] + (defaults: budget 200, experiments/bakeoff_innerloop.json) +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import tempfile +import time +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from homemaker import dom, innerloop, oracle # noqa: E402 + +URB = Path("/home/bruno/src/urb") +EX = URB / "examples" / "programme-house" + +FILES = ( + "2f45907abd9accac2a124d311732f749.dom", + "candidate-002.dom", + "c964435454c459f86c3ed9a5a7621132.dom", +) +SEEDS = (0, 1, 2) +CHECKPOINTS = (40, 80, 120, 200) + + +class TracingEvaluator(innerloop.OracleEvaluator): + """Records (cumulative evals, batch-best fitness) after every oracle call.""" + + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.trace: list[tuple[int, float]] = [] + + def evaluate(self, xs): + scores = super().evaluate(xs) + self.trace.append((self.n_evals, max(s.fitness for s in scores))) + return scores + + def best_at(self, budget: int) -> float: + vals = [f for n, f in self.trace if n <= budget] + return max(vals) if vals else float("nan") + + +class _BudgetExhausted(Exception): + pass + + +def nm_search(ev, x0, budget=200, seed=0): + """Multi-start Nelder-Mead: x0 first, random restarts until the budget is + spent. Sequential — every evaluation is its own oracle invocation.""" + from scipy.optimize import minimize + + rng = np.random.default_rng(seed) + n = len(x0) + x = np.clip(np.asarray(x0, dtype=float), innerloop._EPS, 1 - innerloop._EPS) + s = ev.evaluate([x])[0] + best = innerloop.Result( + x=x.copy(), fitness=s.fitness, n_fails=s.n_fails, fail_lines=s.fail_lines, + x0_fitness=s.fitness, x0_n_fails=s.n_fails, n_evals=0, n_oracle_calls=0, + ) + + def f(xi): + if ev.n_evals >= budget: + raise _BudgetExhausted + sc = ev.evaluate([np.asarray(xi, dtype=float)])[0] + if sc.fitness > best.fitness: + best.x = np.asarray(xi, dtype=float).copy() + best.fitness = sc.fitness + best.n_fails = sc.n_fails + best.fail_lines = sc.fail_lines + return -sc.fitness + + start = x.copy() + while ev.n_evals < budget: + try: + minimize( + f, start, method="Nelder-Mead", + bounds=[(innerloop._EPS, 1 - innerloop._EPS)] * n, + options={"maxfev": budget - ev.n_evals, "xatol": 1e-3, "fatol": 1e-10}, + ) + except _BudgetExhausted: + break + start = rng.uniform(0.1, 0.9, n) # restart + + best.n_evals = ev.n_evals + best.n_oracle_calls = ev.n_oracle_calls + return best + + +def compass_ms_search(ev, x0, budget=200, seed=0, n_starts=3): + """Multi-start compass: budget split evenly; first start is x0, the rest + random. compass_search counts against ev.n_evals, so phase budgets are + cumulative caps.""" + rng = np.random.default_rng(seed) + n = len(x0) + best = None + for phase in range(n_starts): + phase_end = ev.n_evals + (budget - ev.n_evals) // (n_starts - phase) + start = np.asarray(x0, dtype=float) if phase == 0 else rng.uniform(0.1, 0.9, n) + r = innerloop.compass_search(ev, start, budget=phase_end, seed=seed + phase) + if best is None or r.fitness > best.fitness: + keep_x0 = best.x0_fitness if best is not None else r.x0_fitness + keep_x0f = best.x0_n_fails if best is not None else r.x0_n_fails + best = r + best.x0_fitness, best.x0_n_fails = keep_x0, keep_x0f + if ev.n_evals >= budget: + break + best.n_evals = ev.n_evals + best.n_oracle_calls = ev.n_oracle_calls + return best + + +METHODS = { + "nm": nm_search, + "cma": innerloop.cma_search, + "compass": innerloop.compass_search, + "compass-ms": compass_ms_search, +} + + +def main() -> int: + budget = int(sys.argv[1]) if len(sys.argv) > 1 else 200 + out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else ( + Path(__file__).parent / "bakeoff_innerloop.json") + os.environ["URB_NO_OCCLUSION"] = "1" + checkpoints = [c for c in CHECKPOINTS if c <= budget] or [budget] + if checkpoints[-1] != budget: + checkpoints.append(budget) + + # Baselines: the UNMODIFIED originals (gains measured from there, not from + # the equal-offset projection — accept_innerloop.py convention). + orig: dict[str, oracle.Score] = {} + with tempfile.TemporaryDirectory(prefix="bakeoff_orig_") as tmp: + shutil.copy(EX / "patterns.config", tmp) + for name in FILES: + orig[name] = oracle.score(shutil.copy(EX / name, tmp), URB) + + runs = [] + for name in FILES: + for method in METHODS: + for seed in SEEDS: + root = dom.load(str(EX / name)) + with TracingEvaluator(root, EX, URB) as ev: + x0 = ev.x_current + t0 = time.perf_counter() + r = METHODS[method](ev, x0, budget=budget, seed=seed) + dt = time.perf_counter() - t0 + run = { + "file": name, "method": method, "seed": seed, + "dof": len(x0), + "orig_fitness": orig[name].fitness, + "orig_n_fails": orig[name].n_fails, + "x0_fitness": r.x0_fitness, "x0_n_fails": r.x0_n_fails, + "best_at": {str(c): ev.best_at(c) for c in checkpoints}, + "final_fitness": r.fitness, "final_n_fails": r.n_fails, + "n_evals": ev.n_evals, "n_oracle_calls": ev.n_oracle_calls, + "wall_s": dt, + } + runs.append(run) + gains = " ".join( + f"@{c}:x{run['best_at'][str(c)] / orig[name].fitness:.2f}" + for c in checkpoints) + print( + f"{name[:12]:12s} {method:10s} seed={seed} {gains} " + f"fails {orig[name].n_fails}->{r.n_fails} " + f"{ev.n_evals}ev/{ev.n_oracle_calls}calls {dt:.0f}s", + flush=True, + ) + + out_path.write_text(json.dumps( + {"budget": budget, "checkpoints": checkpoints, "runs": runs}, indent=1)) + print(f"\nwrote {out_path}") + + # Summary: mean gain over original at each checkpoint, mean s/eval. + print(f"\n{'method':10s} " + "".join(f"{'x@' + str(c):>8s}" for c in checkpoints) + + f"{'s/eval':>8s}{'calls':>7s}{'fails+':>7s}") + for method in METHODS: + rs = [r for r in runs if r["method"] == method] + cols = "" + for c in checkpoints: + g = np.mean([r["best_at"][str(c)] / r["orig_fitness"] for r in rs]) + cols += f"{g:8.2f}" + spe = np.mean([r["wall_s"] / r["n_evals"] for r in rs]) + calls = np.mean([r["n_oracle_calls"] for r in rs]) + newf = sum(r["final_n_fails"] > r["orig_n_fails"] for r in rs) + print(f"{method:10s} {cols}{spe:8.2f}{calls:7.0f}{newf:7d}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())