bruno/homemaker-layout
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
homemaker-layout/experiments/bakeoff_innerloop.json

767 lines
18 KiB
JSON
Raw Normal View History

Bake-off: CMA-ES confirmed as inner-loop optimiser (homemaker-py-d0s) 4-way comparison (NM / CMA-ES / compass / compass-ms) over 3 corpus files × 3 seeds at budget 200, cold-start, URB_NO_OCCLUSION=1. CMA-ES wins on batch-efficiency (18 oracle calls vs 200 for NM, 12x speedup on Perl startup amortisation per §4.6) with acceptable quality (x1.41 @200 vs NM's x1.56). Compass stalls on narrow-valley landscapes and introduces fail regressions. NM flagged as Phase 3+ candidate once native fitness removes oracle call overhead. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-13 09:46:04 +01:00
{
"budget": 200,
"checkpoints": [
40,
80,
120,
200
],
"runs": [
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "nm",
"seed": 0,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.015336134791349723,
"80": 0.01625453592736873,
"120": 0.016813214603667533,
"200": 0.016970841073693777
},
"final_fitness": 0.016970841073693777,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 426.10284887799935
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "nm",
"seed": 1,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.015336134791349723,
"80": 0.01625453592736873,
"120": 0.016813214603667533,
"200": 0.016970841073693777
},
"final_fitness": 0.016970841073693777,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 407.97355182700267
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "nm",
"seed": 2,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.015336134791349723,
"80": 0.01625453592736873,
"120": 0.016813214603667533,
"200": 0.016970841073693777
},
"final_fitness": 0.016970841073693777,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 410.4154980900057
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "cma",
"seed": 0,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.013602853938370742,
"80": 0.016553033822579822,
"120": 0.016553033822579822,
"200": 0.016553033822579822
},
"final_fitness": 0.02127655034590672,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 350.45485657599056
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "cma",
"seed": 1,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.01095718684601756,
"80": 0.02048062509654076,
"120": 0.02048062509654076,
"200": 0.02048062509654076
},
"final_fitness": 0.02048062509654076,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 347.38566490099765
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "cma",
"seed": 2,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.011174318189865662,
"80": 0.011174318189865662,
"120": 0.018967095390790795,
"200": 0.018967095390790795
},
"final_fitness": 0.018967095390790795,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 337.9982590199943
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass",
"seed": 0,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.006607488139901861,
"80": 0.006607488139901861,
"120": 0.007209518879898406,
"200": 0.008009790075052892
},
"final_fitness": 0.008278786574187515,
"final_n_fails": 3,
"n_evals": 221,
"n_oracle_calls": 11,
"wall_s": 368.2416331849963
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass",
"seed": 1,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.006607488139901861,
"80": 0.006607488139901861,
"120": 0.007170981596472924,
"200": 0.008175449506286243
},
"final_fitness": 0.008399547907246088,
"final_n_fails": 3,
"n_evals": 221,
"n_oracle_calls": 11,
"wall_s": 380.4755447469943
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass",
"seed": 2,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.006607488139901861,
"80": 0.006607488139901861,
"120": 0.006912734615831356,
"200": 0.008078395888024203
},
"final_fitness": 0.008260349591825748,
"final_n_fails": 3,
"n_evals": 221,
"n_oracle_calls": 11,
"wall_s": 369.9571806669992
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass-ms",
"seed": 0,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.006607488139901861,
"80": 0.006607488139901861,
"120": 0.006607488139901861,
"200": 0.006607488139901861
},
"final_fitness": 0.006607488139901861,
"final_n_fails": 3,
"n_evals": 202,
"n_oracle_calls": 12,
"wall_s": 329.4596275669901
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass-ms",
"seed": 1,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.006607488139901861,
"80": 0.006607488139901861,
"120": 0.006607488139901861,
"200": 0.006607488139901861
},
"final_fitness": 0.006607488139901861,
"final_n_fails": 3,
"n_evals": 202,
"n_oracle_calls": 12,
"wall_s": 331.6751732940029
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass-ms",
"seed": 2,
"dof": 7,
"orig_fitness": 0.01303535257266566,
"orig_n_fails": 2,
"x0_fitness": 0.006460366122034485,
"x0_n_fails": 3,
"best_at": {
"40": 0.006607488139901861,
"80": 0.006607488139901861,
"120": 0.006607488139901861,
"200": 0.006607488139901861
},
"final_fitness": 0.006607488139901861,
"final_n_fails": 3,
"n_evals": 202,
"n_oracle_calls": 12,
"wall_s": 353.22266581900476
},
{
"file": "candidate-002.dom",
"method": "nm",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.013043378536981178,
"80": 0.013162194510538643,
"120": 0.013541863507298595,
"200": 0.013861258388999518
},
"final_fitness": 0.013861258388999518,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 533.6580855659995
},
{
"file": "candidate-002.dom",
"method": "nm",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.013043378536981178,
"80": 0.013162194510538643,
"120": 0.013541863507298595,
"200": 0.013861258388999518
},
"final_fitness": 0.013861258388999518,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 534.7732095019892
},
{
"file": "candidate-002.dom",
"method": "nm",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.013043378536981178,
"80": 0.013162194510538643,
"120": 0.013541863507298595,
"200": 0.013861258388999518
},
"final_fitness": 0.013861258388999518,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 529.9340490540053
},
{
"file": "candidate-002.dom",
"method": "cma",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.008766176333159788,
"80": 0.011349351963634282,
"120": 0.01183788578257949,
"200": 0.01183788578257949
},
"final_fitness": 0.01183788578257949,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 478.9317944670038
},
{
"file": "candidate-002.dom",
"method": "cma",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.011654770458049522,
"80": 0.011654770458049522,
"120": 0.011654770458049522,
"200": 0.011654770458049522
},
"final_fitness": 0.011654770458049522,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 486.4617367609899
},
{
"file": "candidate-002.dom",
"method": "cma",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.007280771403204401,
"80": 0.010704424796621367,
"120": 0.011790159199403088,
"200": 0.011790159199403088
},
"final_fitness": 0.011790159199403088,
"final_n_fails": 1,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 486.73567426699447
},
{
"file": "candidate-002.dom",
"method": "compass",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.004059651815091663,
"80": 0.01100524588060076,
"120": 0.013031039518511447,
"200": 0.014373318674033223
},
"final_fitness": 0.01473833833441545,
"final_n_fails": 2,
"n_evals": 211,
"n_oracle_calls": 12,
"wall_s": 502.3361668680009
},
{
"file": "candidate-002.dom",
"method": "compass",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.00921749875086076,
"80": 0.01007215847138918,
"120": 0.010784398878431061,
"200": 0.013581520956957
},
"final_fitness": 0.014220594074265345,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 482.41149059700547
},
{
"file": "candidate-002.dom",
"method": "compass",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.004120081557419885,
"80": 0.005109588901973775,
"120": 0.010453965023981411,
"200": 0.014012902439663506
},
"final_fitness": 0.014332429744919455,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 482.2521033469966
},
{
"file": "candidate-002.dom",
"method": "compass-ms",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.004059651815091663,
"80": 0.01100524588060076,
"120": 0.01100524588060076,
"200": 0.01100524588060076
},
"final_fitness": 0.01100524588060076,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 14,
"wall_s": 333.32965936099936
},
{
"file": "candidate-002.dom",
"method": "compass-ms",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.00921749875086076,
"80": 0.01007215847138918,
"120": 0.01007215847138918,
"200": 0.01007215847138918
},
"final_fitness": 0.01007215847138918,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 14,
"wall_s": 401.0064252880111
},
{
"file": "candidate-002.dom",
"method": "compass-ms",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008075772316711586,
"orig_n_fails": 2,
"x0_fitness": 0.004059651815091663,
"x0_n_fails": 3,
"best_at": {
"40": 0.004120081557419885,
"80": 0.005109588901973775,
"120": 0.005109588901973775,
"200": 0.005109588901973775
},
"final_fitness": 0.005109588901973775,
"final_n_fails": 3,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 282.3956634509959
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "nm",
"seed": 0,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.006177486909077849,
"80": 0.006527788282834294,
"120": 0.0066662305157972855,
"200": 0.006693148459352215
},
"final_fitness": 0.006693148459352215,
"final_n_fails": 3,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 287.4831781459943
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "nm",
"seed": 1,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.006177486909077849,
"80": 0.006527788282834294,
"120": 0.0066662305157972855,
"200": 0.006693148459352215
},
"final_fitness": 0.006693148459352215,
"final_n_fails": 3,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 276.6431823440071
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "nm",
"seed": 2,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.006177486909077849,
"80": 0.006527788282834294,
"120": 0.0066662305157972855,
"200": 0.006693148459352213
},
"final_fitness": 0.006693148459352213,
"final_n_fails": 3,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 275.51684825500706
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "cma",
"seed": 0,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.00533036008277192,
"80": 0.00533036008277192,
"120": 0.00533036008277192,
"200": 0.00533036008277192
},
"final_fitness": 0.00533036008277192,
"final_n_fails": 3,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 225.46989931999997
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "cma",
"seed": 1,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.004648328603302138,
"80": 0.005836221739161555,
"120": 0.005836221739161555,
"200": 0.005836221739161555
},
"final_fitness": 0.005836221739161555,
"final_n_fails": 3,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 227.0608354230062
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "cma",
"seed": 2,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.004653776166165647,
"80": 0.004808176606999774,
"120": 0.004808176606999774,
"200": 0.004808176606999774
},
"final_fitness": 0.004808176606999774,
"final_n_fails": 3,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 226.25115464300325
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass",
"seed": 0,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.002909077000944207,
"80": 0.005636022639392401,
"120": 0.005637234383710328,
"200": 0.006368553115957026
},
"final_fitness": 0.006368553115957026,
"final_n_fails": 3,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 228.6235062899941
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass",
"seed": 1,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.0038443413369024806,
"80": 0.004505393404296923,
"120": 0.005335631011587571,
"200": 0.007639377813299466
},
"final_fitness": 0.0077190451924048515,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 230.63287285399565
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass",
"seed": 2,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.003989234037259822,
"80": 0.003993271664133415,
"120": 0.0083599248344224,
"200": 0.011177097393828111
},
"final_fitness": 0.011177097393828111,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 228.19498966900574
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass-ms",
"seed": 0,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.002909077000944207,
"80": 0.005636022639392401,
"120": 0.005636022639392401,
"200": 0.005636022639392401
},
"final_fitness": 0.005636022639392401,
"final_n_fails": 3,
"n_evals": 213,
"n_oracle_calls": 14,
"wall_s": 222.05841742099437
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass-ms",
"seed": 1,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.0038443413369024806,
"80": 0.004505393404296923,
"120": 0.004505393404296923,
"200": 0.004505393404296923
},
"final_fitness": 0.004505393404296923,
"final_n_fails": 2,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 228.25147005799226
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass-ms",
"seed": 2,
"dof": 6,
"orig_fitness": 0.0039977449092860615,
"orig_n_fails": 3,
"x0_fitness": 0.0019865661983037492,
"x0_n_fails": 4,
"best_at": {
"40": 0.003989234037259822,
"80": 0.003993271664133415,
"120": 0.003993271664133415,
"200": 0.003993271664133415
},
"final_fitness": 0.003993271664133415,
"final_n_fails": 3,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 223.43627467899933
}
]
}
Reference in a new issue Copy permalink