From 3ccbba72841b9beb8c8918063f860a678cb6e68c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 6 Sep 2026 20:58:05 +0000 Subject: [PATCH] Coldstart runner: checkpoint each run, so losing the box costs hours not days The runner never passed --checkpoint-every, so each run's only output landed at the end. The longest single run in the first baseline took 62 h; losing the machine at hour 61 lost all of it. That flag was added during this session precisely for this case and then not wired into the one script that needs it. Default is budget/20 -- 25000 evals at the 500k baseline budget, roughly every 3 h for the slowest programme. --checkpoint-every overrides it. Also documented --slots: it is the number of concurrent runs, each single-worker (one worker per run avoids b8g's parallel non-determinism), so it should match the core count. Verified end to end with a 400-eval run: the .dom.checkpoint appears alongside the output, and the score/record/push path still works. The smoke-test artefacts were removed rather than committed. Refs homemaker-py-bk9. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB --- experiments/run_coldstart_baseline.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/experiments/run_coldstart_baseline.py b/experiments/run_coldstart_baseline.py index 8dd3b4d..7130761 100644 --- a/experiments/run_coldstart_baseline.py +++ b/experiments/run_coldstart_baseline.py @@ -122,15 +122,25 @@ def main() -> None: formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--budget", type=int, default=500000) ap.add_argument("--seeds", type=int, default=3) - ap.add_argument("--slots", type=int, default=4) + ap.add_argument("--slots", type=int, default=4, + help="concurrent runs; one worker each, so set it to your " + "core count (default 4)") + ap.add_argument("--checkpoint-every", type=int, default=None, metavar="N", + help="write each run's best-so-far .dom every N evals " + "(default: budget/20). The longest single run in the " + "first baseline took 62 h; without this, losing the " + "box at hour 61 loses all of it.") ap.add_argument("--programmes", nargs="+", default=PROGRAMMES) ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() + checkpoint_every = (args.checkpoint_every if args.checkpoint_every is not None + else max(1, args.budget // 20)) # seed-major: all programmes at seed 0, then seed 1, ... queue = [(p, s) for s in range(args.seeds) for p in args.programmes] print(f"{len(queue)} runs, budget {args.budget}, {args.slots} slots, " - f"seed-major order\n", flush=True) + f"checkpoint every {checkpoint_every} evals, seed-major order\n", + flush=True) if args.dry_run: for p, s in queue: print(f" would run {p} seed {s}") @@ -146,7 +156,8 @@ def main() -> None: fh = log.open("w") proc = subprocess.Popen( ["homemaker-evolve", "init.dom", "--budget", str(args.budget), - "--seed", str(seed), "--workers", "1", "--output", str(out)], + "--seed", str(seed), "--workers", "1", "--output", str(out), + "--checkpoint-every", str(checkpoint_every)], cwd=d, stdout=subprocess.DEVNULL, stderr=fh) running[proc.pid] = (proc, prog, seed, out, fh, time.time()) print(f" start {prog} seed {seed} -> {out.name}", flush=True)