bruno/homemaker-layout
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
homemaker-layout/experiments/bakeoff_native.json

767 lines
18 KiB
JSON
Raw Normal View History

Swap inner loop default from CMA-ES to Nelder-Mead (homemaker-py-d6d) Bakeoff with native fitness shows NM wins at all DOF sizes: +9% at child_budget=80 for programme-house (6-7 DOF), and decisively at harbor-house scale (35-40 DOF) where CMA-ES exhausts its convergence detector after ~3 generations (46 evals) and adds failures on 12/15 runs. NM uses the full budget, is parameter-free, and has zero new failures across all test cases. - Add nm_search() to innerloop.py; change optimise() default to "nm" - Add nm_search to parametrised test cases - Add bakeoff_native.py and bakeoff_harbor.py experiments with results Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-14 08:51:22 +01:00
{
"budget": 200,
"checkpoints": [
40,
80,
120,
200
],
"runs": [
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "nm",
"seed": 0,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.015757102927373864,
"80": 0.016518999094772138,
"120": 0.01704550937987012,
"200": 0.017095573931846186
},
"final_fitness": 0.017095573931846186,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 3.03920799199841
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "nm",
"seed": 1,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.015757102927373864,
"80": 0.016518999094772138,
"120": 0.01704550937987012,
"200": 0.017095573931846186
},
"final_fitness": 0.017095573931846186,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 3.1659032219904475
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "nm",
"seed": 2,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.015757102927373864,
"80": 0.016518999094772138,
"120": 0.01704550937987012,
"200": 0.017095573931846186
},
"final_fitness": 0.017095573931846186,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 3.05030325500411
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "cma",
"seed": 0,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.010663621166521513,
"80": 0.016021190997154953,
"120": 0.016021190997154953,
"200": 0.01911100059868815
},
"final_fitness": 0.01911100059868815,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 4.800741150014801
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "cma",
"seed": 1,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.014245337526455628,
"80": 0.01713451710200977,
"120": 0.01713451710200977,
"200": 0.01713451710200977
},
"final_fitness": 0.01713451710200977,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 3.216690313012805
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "cma",
"seed": 2,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.015440558283110382,
"80": 0.019877986127931697,
"120": 0.019877986127931697,
"200": 0.019877986127931697
},
"final_fitness": 0.019877986127931697,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 3.184704232000513
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass",
"seed": 0,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.006788334438689785,
"80": 0.006788334438689785,
"120": 0.007412193161228044,
"200": 0.008240100514146836
},
"final_fitness": 0.008507130978612422,
"final_n_fails": 3,
"n_evals": 221,
"n_oracle_calls": 11,
"wall_s": 3.304916718014283
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass",
"seed": 1,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.006788334438689785,
"80": 0.006788334438689785,
"120": 0.007377140294536451,
"200": 0.008405833810967343
},
"final_fitness": 0.008630232815926193,
"final_n_fails": 3,
"n_evals": 221,
"n_oracle_calls": 11,
"wall_s": 3.335104385012528
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass",
"seed": 2,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.006788334438689785,
"80": 0.006788334438689785,
"120": 0.007117081498828506,
"200": 0.008302840911437218
},
"final_fitness": 0.008491968813582865,
"final_n_fails": 3,
"n_evals": 221,
"n_oracle_calls": 11,
"wall_s": 3.3419314719794784
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass-ms",
"seed": 0,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.006788334438689785,
"80": 0.006788334438689785,
"120": 0.006788334438689785,
"200": 0.006788334438689785
},
"final_fitness": 0.006788334438689785,
"final_n_fails": 3,
"n_evals": 202,
"n_oracle_calls": 12,
"wall_s": 2.9742391599866096
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass-ms",
"seed": 1,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.006788334438689785,
"80": 0.006788334438689785,
"120": 0.006788334438689785,
"200": 0.006788334438689785
},
"final_fitness": 0.006788334438689785,
"final_n_fails": 3,
"n_evals": 202,
"n_oracle_calls": 12,
"wall_s": 2.9756440880009905
},
{
"file": "2f45907abd9accac2a124d311732f749.dom",
"method": "compass-ms",
"seed": 2,
"dof": 7,
"orig_fitness": 0.013382088019204255,
"orig_n_fails": 2,
"x0_fitness": 0.00663720042265933,
"x0_n_fails": 3,
"best_at": {
"40": 0.006788334438689785,
"80": 0.006788334438689785,
"120": 0.006788334438689785,
"200": 0.006788334438689785
},
"final_fitness": 0.006788334438689785,
"final_n_fails": 3,
"n_evals": 202,
"n_oracle_calls": 12,
"wall_s": 2.9858868430019356
},
{
"file": "candidate-002.dom",
"method": "nm",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.012812479126328056,
"80": 0.013473373465090525,
"120": 0.01370648588540478,
"200": 0.01417274835180044
},
"final_fitness": 0.01417274835180044,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 3.1041710549907293
},
{
"file": "candidate-002.dom",
"method": "nm",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.012812479126328056,
"80": 0.013473373465090525,
"120": 0.01370648588540478,
"200": 0.01417274835180044
},
"final_fitness": 0.01417274835180044,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 3.1137547309917863
},
{
"file": "candidate-002.dom",
"method": "nm",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.012812479126328056,
"80": 0.013473373465090525,
"120": 0.01370648588540478,
"200": 0.01417274835180044
},
"final_fitness": 0.01417274835180044,
"final_n_fails": 2,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 3.127368861983996
},
{
"file": "candidate-002.dom",
"method": "cma",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.012007007539014592,
"80": 0.012007007539014592,
"120": 0.012007007539014592,
"200": 0.012007007539014592
},
"final_fitness": 0.012007007539014592,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 3.3279110910079908
},
{
"file": "candidate-002.dom",
"method": "cma",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.01186649407002513,
"80": 0.01186649407002513,
"120": 0.017038736223641965,
"200": 0.017038736223641965
},
"final_fitness": 0.017038736223641965,
"final_n_fails": 1,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 3.270579600997735
},
{
"file": "candidate-002.dom",
"method": "cma",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.010098944430771193,
"80": 0.015379247042619166,
"120": 0.015379247042619166,
"200": 0.015379247042619166
},
"final_fitness": 0.015525042109130723,
"final_n_fails": 1,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 3.3238525039923843
},
{
"file": "candidate-002.dom",
"method": "compass",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.004102818811068907,
"80": 0.011183119580128308,
"120": 0.01320274831431111,
"200": 0.014620147366591679
},
"final_fitness": 0.014992004956445376,
"final_n_fails": 2,
"n_evals": 211,
"n_oracle_calls": 12,
"wall_s": 3.2189600660058204
},
{
"file": "candidate-002.dom",
"method": "compass",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.009433202521971773,
"80": 0.015938032135346045,
"120": 0.01635093680800657,
"200": 0.01918644136301778
},
"final_fitness": 0.01918644136301778,
"final_n_fails": 1,
"n_evals": 211,
"n_oracle_calls": 12,
"wall_s": 3.264885897981003
},
{
"file": "candidate-002.dom",
"method": "compass",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.0041820098440464816,
"80": 0.005189753198720991,
"120": 0.010694625638119038,
"200": 0.014250388767450632
},
"final_fitness": 0.014527892401089936,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 3.4018315189750865
},
{
"file": "candidate-002.dom",
"method": "compass-ms",
"seed": 0,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.004102818811068907,
"80": 0.011183119580128308,
"120": 0.011183119580128308,
"200": 0.011183119580128308
},
"final_fitness": 0.011183119580128308,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 14,
"wall_s": 3.1870358180021867
},
{
"file": "candidate-002.dom",
"method": "compass-ms",
"seed": 1,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.009433202521971773,
"80": 0.015938032135346045,
"120": 0.015938032135346045,
"200": 0.015938032135346045
},
"final_fitness": 0.015938032135346045,
"final_n_fails": 1,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 3.264965091017075
},
{
"file": "candidate-002.dom",
"method": "compass-ms",
"seed": 2,
"dof": 6,
"orig_fitness": 0.008168633107088569,
"orig_n_fails": 2,
"x0_fitness": 0.004102818811068907,
"x0_n_fails": 3,
"best_at": {
"40": 0.0041820098440464816,
"80": 0.005189753198720991,
"120": 0.005189753198720991,
"200": 0.005189753198720991
},
"final_fitness": 0.005189753198720991,
"final_n_fails": 3,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 3.231278574006865
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "nm",
"seed": 0,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.00628116027564786,
"80": 0.0066772033854169775,
"120": 0.006780624745158266,
"200": 0.006810329001098464
},
"final_fitness": 0.006810329001098464,
"final_n_fails": 3,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 2.8260676650097594
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "nm",
"seed": 1,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.00628116027564786,
"80": 0.0066772033854169775,
"120": 0.006780624745158266,
"200": 0.006810329001098464
},
"final_fitness": 0.006810329001098464,
"final_n_fails": 3,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 2.7759857880009804
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "nm",
"seed": 2,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.00628116027564786,
"80": 0.0066772033854169775,
"120": 0.006780624745158266,
"200": 0.006810329001098464
},
"final_fitness": 0.006810329001098464,
"final_n_fails": 3,
"n_evals": 200,
"n_oracle_calls": 200,
"wall_s": 2.801363606005907
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "cma",
"seed": 0,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.004600448055289459,
"80": 0.005250603330111013,
"120": 0.005560282616538349,
"200": 0.005560282616538349
},
"final_fitness": 0.005560282616538349,
"final_n_fails": 3,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 2.9268163639935665
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "cma",
"seed": 1,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.0037649133919586994,
"80": 0.00591003675413222,
"120": 0.009857950172868,
"200": 0.009857950172868
},
"final_fitness": 0.009857950172868,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 2.9385673919750843
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "cma",
"seed": 2,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.0034062417089593166,
"80": 0.0037385412442699213,
"120": 0.0054098389503915816,
"200": 0.005440225403774762
},
"final_fitness": 0.005440225403774762,
"final_n_fails": 2,
"n_evals": 208,
"n_oracle_calls": 18,
"wall_s": 2.9514843790093437
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass",
"seed": 0,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.002248659154845088,
"80": 0.0032413748551064557,
"120": 0.006288102257076037,
"200": 0.008568542752225914
},
"final_fitness": 0.008947615284511726,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 2.926597241021227
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass",
"seed": 1,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.002248659154845088,
"80": 0.0046341557697230475,
"120": 0.004820786067703021,
"200": 0.009098891067442778
},
"final_fitness": 0.009129274342508454,
"final_n_fails": 2,
"n_evals": 215,
"n_oracle_calls": 12,
"wall_s": 2.8956846530199982
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass",
"seed": 2,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.002248659154845088,
"80": 0.007786433996865836,
"120": 0.0078040098047759444,
"200": 0.008913724662330083
},
"final_fitness": 0.009448457212325586,
"final_n_fails": 2,
"n_evals": 213,
"n_oracle_calls": 12,
"wall_s": 2.9612420260091312
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass-ms",
"seed": 0,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.002248659154845088,
"80": 0.0032413748551064557,
"120": 0.0032413748551064557,
"200": 0.0032413748551064557
},
"final_fitness": 0.0032413748551064557,
"final_n_fails": 2,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 2.925349555996945
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass-ms",
"seed": 1,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.002248659154845088,
"80": 0.0046341557697230475,
"120": 0.0046341557697230475,
"200": 0.0046341557697230475
},
"final_fitness": 0.0046341557697230475,
"final_n_fails": 2,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 2.9594067580183037
},
{
"file": "c964435454c459f86c3ed9a5a7621132.dom",
"method": "compass-ms",
"seed": 2,
"dof": 6,
"orig_fitness": 0.004043222043120658,
"orig_n_fails": 3,
"x0_fitness": 0.002007478958321212,
"x0_n_fails": 4,
"best_at": {
"40": 0.002248659154845088,
"80": 0.007786433996865836,
"120": 0.007786433996865836,
"200": 0.007786433996865836
},
"final_fitness": 0.007786433996865836,
"final_n_fails": 2,
"n_evals": 215,
"n_oracle_calls": 14,
"wall_s": 2.906411771022249
}
]
}
Reference in a new issue Copy permalink