From e5eb397b521da6feb8592e17ab242465bcf9125d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 05:43:58 +0000 Subject: [PATCH] Checkpoint long searches; the cold-start runs were lost to a reclaimed box All four 500k runs died about 10 minutes in when the container was reclaimed. No SIGTERM fired, so no .dom was written and 0 of 12 runs completed. My plan committed results per finished run, which protected nothing because no run reached its commit point. The bad assumption was reading "reclaimed after inactivity" as CPU inactivity; it is conversation inactivity, and background compute does not hold the box open. Progress reached before the loss (from the tracked logs): harbor 24,960 evals / 40 fails, maple 14,880 / 79, health-centre 25,920 / 33, programme-house 138,800 / 2. The underlying gap is not environmental: a search's only output lands at the very end or on SIGTERM, so ANY abrupt loss -- reclaimed container, OOM, power cut -- takes the whole run with it. On a 3M-eval search that is 2.4 days of compute with no recoverable artefact. - driver.search gains checkpoint=/checkpoint_every=: the current best is handed to a callback at most every N evals. Rate-limited by evals, not improvements, which come in bursts early. A failing checkpoint is logged and swallowed -- losing a checkpoint is bad, losing the search because a checkpoint failed is worse. - homemaker-evolve --checkpoint-every N writes .dom.checkpoint via mkstemp + os.replace, so a crash can never catch it half-written. It is deliberately NOT the output path: a checkpoint is a leaf-sharing run's internal best, dishonest under the canonical scorer until the finish stage unfolds it (homemaker-py-3l6), and must not be mistaken for the finished article. - Verified the written checkpoint re-loads as a valid .dom. Default off, so behaviour is unchanged without the flag. Lint at parity (46); tests 372 passed (3 new), same 2 pre-existing failures. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB --- examples/harbor-house/coldstart-500000-s0.log | 20 +++++++++ .../health-centre/coldstart-500000-s0.log | 36 +++++++++++++++ examples/maple-court/coldstart-500000-s0.log | 21 +++++++++ .../programme-house/coldstart-500000-s0.log | 31 +++++++++++++ experiments/results/coldstart_baseline.tsv | 1 - src/homemaker_layout/driver.py | 19 ++++++++ src/homemaker_layout/evolve.py | 44 ++++++++++++++++++ tests/test_driver.py | 45 +++++++++++++++++++ 8 files changed, 216 insertions(+), 1 deletion(-) diff --git a/examples/harbor-house/coldstart-500000-s0.log b/examples/harbor-house/coldstart-500000-s0.log index a14c109..083e317 100644 --- a/examples/harbor-house/coldstart-500000-s0.log +++ b/examples/harbor-house/coldstart-500000-s0.log @@ -21,3 +21,23 @@ output : /home/user/homemaker-layout/examples/harbor-house/coldstart-50000 [ 240 evals] best 6.52063e-20 (fails 55) via construct/2 [ 1280 evals] best 3.07967e-20 (fails 54) via construct/15 [ 1760 evals] best 1.59184e-19 (fails 54) via swap 0/lrr +[ 1920 evals] best 6.67908e-19 (fails 52) via rotate 0/lrll +[ 3120 evals] best 1.30712e-18 (fails 51) via core_undivide noop +[ 4400 evals] best 1.39275e-18 (fails 51) via core_divide noop +[ 6720 evals] best 1.41743e-18 (fails 51) via swap 0/lrr +[ 8640 evals] best 2.87976e-18 (fails 50) via core_undivide noop +[ 10000 evals] best 5.71109e-18 (fails 49) via rotate 0/rlll +[ 12240 evals] best 5.95838e-18 (fails 49) via crossover lrlll<->lrlll +[ 12400 evals] best 1.17875e-17 (fails 48) via level_compound_fix noop +[ 12880 evals] best 1.24564e-17 (fails 48) via level_fix noop +[ 15120 evals] best 1.40283e-17 (fails 48) via swap 0/llr +[ 16080 evals] best 1.44296e-17 (fails 48) via level_compound_fix noop +[ 17600 evals] best 1.48821e-16 (fails 48) via swap 0/llr +[ 18960 evals] best 5.32871e-16 (fails 42) via crossover lrllr<->rrl +[ 21680 evals] best 1.25e-15 (fails 41) via crossover rlrrr<->llll +[ 22720 evals] best 3.08937e-15 (fails 40) via retype 0/lrrr->m +[ 22800 evals] best 3.1408e-15 (fails 40) via core_undivide noop +[ 23280 evals] best 3.18171e-15 (fails 40) via level_fix noop +[ 24000 evals] best 3.39655e-15 (fails 40) via level_compound_fix noop +[ 24160 evals] best 3.42463e-15 (fails 40) via level_compound_fix noop +[ 24960 evals] best 3.43166e-15 (fails 40) via level_compound_fix noop diff --git a/examples/health-centre/coldstart-500000-s0.log b/examples/health-centre/coldstart-500000-s0.log index 4a39793..0c82c8d 100644 --- a/examples/health-centre/coldstart-500000-s0.log +++ b/examples/health-centre/coldstart-500000-s0.log @@ -20,3 +20,39 @@ shapecurve prune : False output : /home/user/homemaker-layout/examples/health-centre/coldstart-500000-s0.dom [ 80 evals] best 7.05564e-20 (fails 53) via construct/0 [ 400 evals] best 1.03209e-17 (fails 47) via construct/4 +[ 2160 evals] best 8.5085e-17 (fails 44) via level_compound_fix noop +[ 2800 evals] best 8.71761e-17 (fails 44) via level_retype noop +[ 3280 evals] best 1.01747e-16 (fails 44) via place_missing t9 -> llrrr +[ 3760 evals] best 1.07254e-16 (fails 44) via core_divide noop +[ 3840 evals] best 1.94754e-16 (fails 43) via retype 0/llrrl->ph1 +[ 4800 evals] best 2.09073e-16 (fails 42) via core_divide noop +[ 4960 evals] best 2.709e-16 (fails 42) via crossover lrlll<->rrrrr +[ 5840 evals] best 2.7669e-16 (fails 42) via crossover lllll<->rllll +[ 5920 evals] best 9.22724e-16 (fails 41) via place_missing t9 -> rrrlr +[ 6480 evals] best 9.45485e-16 (fails 41) via place_missing noop +[ 6960 evals] best 3.7158e-15 (fails 39) via place_missing t9 -> rrrlr +[ 8320 evals] best 4.98363e-15 (fails 39) via level_compound_fix noop +[ 9680 evals] best 5.20512e-15 (fails 39) via crossover lrlll<->rlrrl +[ 9840 evals] best 5.48608e-15 (fails 39) via place_missing noop +[ 10080 evals] best 5.67651e-15 (fails 39) via crossover rrrrr<->rllrl +[ 10640 evals] best 6.58735e-15 (fails 38) via retype 0/rrllr->O +[ 12320 evals] best 7.02146e-15 (fails 38) via level_fix noop +[ 12880 evals] best 7.53327e-15 (fails 38) via level_compound_fix noop +[ 14240 evals] best 8.06389e-15 (fails 38) via level_compound_fix noop +[ 14480 evals] best 1.56759e-14 (fails 37) via swap 0/llrl +[ 14560 evals] best 1.59668e-14 (fails 37) via crossover rrlrl<->llrlr +[ 14880 evals] best 1.62722e-14 (fails 37) via core_undivide noop +[ 15360 evals] best 3.83618e-14 (fails 36) via level_retype noop +[ 16320 evals] best 3.97387e-14 (fails 36) via level_fix noop +[ 16560 evals] best 4.10139e-14 (fails 36) via level_fix noop +[ 17040 evals] best 4.70405e-14 (fails 35) via swap 0/llrl +[ 17600 evals] best 5.08607e-14 (fails 35) via level_compound_fix noop +[ 20320 evals] best 9.45373e-14 (fails 35) via crossover rllr<->rlrll +[ 21040 evals] best 9.66604e-14 (fails 35) via level_compound_fix noop +[ 21600 evals] best 9.92079e-14 (fails 35) via level_fix noop +[ 21840 evals] best 1.24031e-13 (fails 35) via level_retype noop +[ 22000 evals] best 1.45873e-13 (fails 35) via crossover lrrl<->lrlrl +[ 23440 evals] best 1.48115e-13 (fails 35) via retype 0/rrlrr->pt1 +[ 23840 evals] best 1.51872e-13 (fails 35) via level_compound_fix noop +[ 25200 evals] best 5.3306e-13 (fails 33) via undivide 0/llll +[ 25920 evals] best 5.46749e-13 (fails 33) via level_fix noop diff --git a/examples/maple-court/coldstart-500000-s0.log b/examples/maple-court/coldstart-500000-s0.log index c8e5d08..d8c3a8d 100644 --- a/examples/maple-court/coldstart-500000-s0.log +++ b/examples/maple-court/coldstart-500000-s0.log @@ -19,3 +19,24 @@ shapecurve prune : False output : /home/user/homemaker-layout/examples/maple-court/coldstart-500000-s0.dom [ 80 evals] best 1.5923e-44 (fails 136) via construct/0 [ 160 evals] best 4.80951e-39 (fails 119) via construct/1 +[ 1360 evals] best 2.33063e-35 (fails 108) via level_compound_fix noop +[ 1600 evals] best 4.62166e-35 (fails 107) via undivide 0/lrrl +[ 2400 evals] best 2.12969e-34 (fails 105) via core_undivide noop +[ 2640 evals] best 8.34232e-34 (fails 103) via crossover lrlrr<->lrrll +[ 3120 evals] best 1.18679e-33 (fails 102) via retype 2/rrrr->ws1 +[ 3520 evals] best 2.83542e-32 (fails 98) via core_undivide noop +[ 4240 evals] best 5.90625e-32 (fails 97) via swap 0/lrlr +[ 4800 evals] best 6.75777e-32 (fails 97) via core_divide lrlr (2 floors) +[ 5120 evals] best 9.68203e-30 (fails 90) via core_divide lrlr (2 floors) +[ 6720 evals] best 1.06032e-29 (fails 90) via crossover lrll<->lrll +[ 6800 evals] best 2.32821e-29 (fails 89) via swap 0/lrrr +[ 7440 evals] best 9.83875e-29 (fails 87) via divide 0/rlll +[ 7520 evals] best 1.98761e-28 (fails 86) via core_undivide noop +[ 8960 evals] best 3.45107e-27 (fails 82) via level_fix ef1: lvl2/lrlrr โ†’ lvl0/rllr +[ 9600 evals] best 3.48472e-27 (fails 82) via core_undivide noop +[ 12320 evals] best 3.50776e-27 (fails 82) via level_compound_fix noop +[ 12640 evals] best 3.62472e-27 (fails 82) via swap 2/rlll +[ 13440 evals] best 3.64565e-27 (fails 82) via core_undivide noop +[ 14320 evals] best 3.67293e-27 (fails 82) via retype 0/rllll->py +[ 14640 evals] best 1.36771e-26 (fails 80) via level_compound_fix py: lvl0 โ†’ lvl1/lrr +[ 14880 evals] best 2.61906e-26 (fails 79) via divide 1/llrr diff --git a/examples/programme-house/coldstart-500000-s0.log b/examples/programme-house/coldstart-500000-s0.log index 1277086..94d19ed 100644 --- a/examples/programme-house/coldstart-500000-s0.log +++ b/examples/programme-house/coldstart-500000-s0.log @@ -30,3 +30,34 @@ output : /home/user/homemaker-layout/examples/programme-house/coldstart-50 [ 7840 evals] best 7.11352e-05 (fails 5) via core_divide noop [ 8400 evals] best 0.000111875 (fails 5) via undivide 0/rr [ 8480 evals] best 0.000114961 (fails 5) via core_undivide noop +[ 10640 evals] best 8.54011e-05 (fails 4) via swap 0/l +[ 11440 evals] best 8.56844e-05 (fails 4) via crossover rlr<->ll +[ 11600 evals] best 8.57076e-05 (fails 4) via level_retype 1/rl<->0/llr +[ 13120 evals] best 8.58391e-05 (fails 4) via core_undivide noop +[ 13440 evals] best 8.73822e-05 (fails 4) via undivide 0/rl +[ 15760 evals] best 0.000112799 (fails 4) via crossover ll<->ll +[ 16320 evals] best 0.00015754 (fails 3) via level_retype 1/lr<->0/rl +[ 16480 evals] best 0.00020643 (fails 3) via swap 0/ll +[ 18480 evals] best 0.000208867 (fails 3) via crossover lll<->llr +[ 19360 evals] best 2.54868e-05 (fails 2) via level_compound_fix l1: lvl1 โ†’ lvl0/rl +[ 19520 evals] best 2.54914e-05 (fails 2) via crossover rr<->rr +[ 19920 evals] best 2.54938e-05 (fails 2) via level_compound_fix noop +[ 22720 evals] best 0.000445739 (fails 2) via undivide 0/rl +[ 25760 evals] best 0.000446136 (fails 2) via crossover lll<->lll +[ 26080 evals] best 0.000446413 (fails 2) via core_undivide noop +[ 28800 evals] best 0.000446749 (fails 2) via retype 1/rrl->b2 +[ 29440 evals] best 0.000447236 (fails 2) via level_retype 1/rrl<->0/rl +[ 29920 evals] best 0.000447713 (fails 2) via crossover r<->r +[ 30320 evals] best 0.00044781 (fails 2) via crossover rrr<->llr +[ 33520 evals] best 0.000447846 (fails 2) via retype 0/lr->t2 +[ 33600 evals] best 0.000447864 (fails 2) via crossover root<->root +[ 92160 evals] best 0.00248135 (fails 2) via retype 1/ll->O +[102080 evals] best 0.00248174 (fails 2) via crossover rrr<->llr +[102880 evals] best 0.00248226 (fails 2) via crossover rrl<->rrl +[103120 evals] best 0.00248262 (fails 2) via retype 1/rl->t2 +[103200 evals] best 0.00248301 (fails 2) via core_undivide noop +[110640 evals] best 0.0024834 (fails 2) via crossover ll<->ll +[111920 evals] best 0.00248376 (fails 2) via crossover root<->root +[114080 evals] best 0.00248391 (fails 2) via level_retype 1/rl<->0/lr +[120240 evals] best 0.00248398 (fails 2) via level_retype 1/rrr<->0/rrr +[138800 evals] best 0.00248406 (fails 2) via level_delete (2 storeys) diff --git a/experiments/results/coldstart_baseline.tsv b/experiments/results/coldstart_baseline.tsv index 5eee2e7..322fcdc 100644 --- a/experiments/results/coldstart_baseline.tsv +++ b/experiments/results/coldstart_baseline.tsv @@ -1,2 +1 @@ programme seed budget fails hard soft score elapsed_s dom -programme-house 0 1000 7 3 4 2.02918e-05 10.0 coldstart-1000-s0.dom diff --git a/src/homemaker_layout/driver.py b/src/homemaker_layout/driver.py index 79c4742..c8f2c68 100644 --- a/src/homemaker_layout/driver.py +++ b/src/homemaker_layout/driver.py @@ -317,6 +317,8 @@ def search( assign_solver: str = "greedy", enable_reassign: bool = False, preserve_circulation: bool = False, + checkpoint=None, + checkpoint_every: int = 0, ) -> SearchResult: """Run the memetic loop from ``seed_root`` until ``budget`` oracle evaluations are consumed. Returns the best individual found; its ``root`` @@ -497,6 +499,8 @@ def search( do_bootstrap = (not seed_root.divided) if bootstrap is None else bootstrap + last_checkpoint = [0] # list so the nested recorder can rebind it + def _log(msg: str) -> None: if log: log(msg) @@ -523,6 +527,21 @@ def search( (n_evals, len({p.sig for p in pop} | {ind.sig}), len(seen_sigs))) _log(f"[{n_evals:6d} evals] best {ind.fitness:.6g} " f"(fails {ind.n_fails}) via {ind.lineage}") + # Crash safety for long runs. A 3M-eval search is days of compute + # whose only output lands at the very end (or on SIGTERM), so an + # abrupt loss -- a reclaimed container, an OOM, a power cut -- + # takes everything with it. When `checkpoint` is given it is + # handed the current best at most every `checkpoint_every` evals, + # so the run always has a recoverable artefact on disk. Rate-limited + # by evals, not by improvements, because improvements come in + # bursts early on. A failing checkpoint must never kill the search. + if checkpoint is not None and ( + n_evals - last_checkpoint[0] >= checkpoint_every): + last_checkpoint[0] = n_evals + try: + checkpoint(result.best, n_evals) + except Exception as exc: # noqa: BLE001 + _log(f"[{n_evals:6d} evals] checkpoint failed: {exc!r}") if niche_by_signature: # ยง11.5 structural niching: at most one individual per topology # signature, keeping the better of any collision. This preserves diff --git a/src/homemaker_layout/evolve.py b/src/homemaker_layout/evolve.py index 80ea5b9..b29a350 100644 --- a/src/homemaker_layout/evolve.py +++ b/src/homemaker_layout/evolve.py @@ -71,6 +71,11 @@ def _parse_args(argv=None) -> argparse.Namespace: p.add_argument("--child-budget", type=int, default=_env_int("HOMEMAKER_CHILD_BUDGET", 80), metavar="N", help="per-child evaluation budget") + p.add_argument("--checkpoint-every", type=int, default=0, metavar="N", + help="write the best-so-far .dom every N evals (0 = off). " + "Crash safety for long runs: without it the only " + "output lands at the end, so a reclaimed container or " + "an OOM loses the whole search.") p.add_argument("--workers", type=int, default=_env_int("HOMEMAKER_WORKERS", 1), metavar="N", help="parallel worker processes") @@ -302,6 +307,41 @@ def main(argv=None) -> int: _preflight(programme_dir) + def _make_checkpoint(path): + """Write best-so-far to `.checkpoint` ATOMICALLY. + + A checkpoint is worthless if a crash can catch it half-written, so it + goes to a temp file in the same directory and is renamed over the + target (rename is atomic within a filesystem). It is deliberately NOT + the final output path -- a checkpoint is a leaf-sharing run's internal + best, which is dishonest under the canonical scorer until the finish + stage unfolds it (homemaker-py-3l6), so it must not be mistaken for + the finished article. + """ + import os + import tempfile + + def _write(best, n_evals): + if best is None: + return + d = os.path.dirname(path) or "." + fd, tmp = tempfile.mkstemp(dir=d, suffix=".ckpt") + try: + with os.fdopen(fd, "w") as fh: + fh.write(dom.dumps(best.root)) + os.replace(tmp, path) + except BaseException: + if os.path.exists(tmp): + os.unlink(tmp) + raise + print(f"[{n_evals:6d} evals] checkpoint -> {os.path.basename(path)} " + f"({best.n_fails} fails)", file=sys.stderr, flush=True) + + return _write + + _ckpt = (_make_checkpoint(str(out) + ".checkpoint") + if args.checkpoint_every > 0 and args.output != Path("-") else None) + print(f"seed : {seed_file}", file=sys.stderr) print(f"programme : {programme_dir.name}", file=sys.stderr) print(f"budget : {args.budget}", file=sys.stderr) @@ -353,6 +393,8 @@ def main(argv=None) -> int: n_workers=args.workers, superpose=args.superpose, multi_use=args.multi_use, + checkpoint=_ckpt, + checkpoint_every=args.checkpoint_every, log=lambda m: print(m, file=sys.stderr, flush=True), ) _finish_sharing = False @@ -377,6 +419,8 @@ def main(argv=None) -> int: collapse_insearch=args.collapse_insearch, shapecurve_warmstart=args.shapecurve_warmstart, shapecurve_prune=args.shapecurve_prune, + checkpoint=_ckpt, + checkpoint_every=args.checkpoint_every, log=lambda m: print(m, file=sys.stderr, flush=True), ) _finish_sharing = args.leaf_sharing diff --git a/tests/test_driver.py b/tests/test_driver.py index a49b49f..4383936 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -672,3 +672,48 @@ def test_use_tiers_prefers_fewer_hard_over_fewer_total_fails(monkeypatch): tiered = driver.search(copy.deepcopy(seed_root), use_tiers=True, **common_kw) assert tiered.best.n_hard == 0 # tiered comparator: fewer hard fails wins assert tiered.best.n_fails == 2 + + +# --------------------------------------------------------------------------- # +# Crash safety for long runs: driver.search's checkpoint hook +# --------------------------------------------------------------------------- # +def test_search_checkpoint_is_called_with_the_current_best(tmp_path): + """A long search's only output otherwise lands at the very end, so an + abrupt loss takes the whole run with it.""" + seen = [] + r = driver.search( + dom.load("examples/programme-house/init.dom"), + "examples/programme-house", budget=400, seed=0, child_budget=20, + checkpoint=lambda best, n: seen.append((n, best.n_fails)), + checkpoint_every=50) + assert seen, "checkpoint was never called" + evals = [n for n, _ in seen] + assert evals == sorted(evals) + assert all(b - a >= 50 for a, b in zip(evals, evals[1:])), \ + "checkpoint_every must rate-limit by evals, not fire on every improvement" + assert r.best is not None + + +def test_search_checkpoint_failure_never_kills_the_run(): + """Losing a checkpoint is bad; losing the search because a checkpoint + failed is worse.""" + def boom(best, n): + raise OSError("disk full") + + r = driver.search( + dom.load("examples/programme-house/init.dom"), + "examples/programme-house", budget=400, seed=0, child_budget=20, + checkpoint=boom, checkpoint_every=50) + assert r.best is not None + + +def test_search_without_checkpoint_is_unchanged(): + """Default off: no hook, byte-identical result to before the feature.""" + kw = dict(budget=400, seed=0, child_budget=20) + a = driver.search(dom.load("examples/programme-house/init.dom"), + "examples/programme-house", **kw) + b = driver.search(dom.load("examples/programme-house/init.dom"), + "examples/programme-house", + checkpoint=None, checkpoint_every=0, **kw) + assert a.best.fitness == b.best.fitness + assert a.n_evals == b.n_evals