bruno/homemaker-layout
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Coldstart runner: checkpoint each run, so losing the box costs hours not days

Browse source 
The runner never passed --checkpoint-every, so each run's only output landed at
the end. The longest single run in the first baseline took 62 h; losing the
machine at hour 61 lost all of it. That flag was added during this session
precisely for this case and then not wired into the one script that needs it.

Default is budget/20 -- 25000 evals at the 500k baseline budget, roughly every
3 h for the slowest programme. --checkpoint-every overrides it.

Also documented --slots: it is the number of concurrent runs, each single-worker
(one worker per run avoids b8g's parallel non-determinism), so it should match
the core count.

Verified end to end with a 400-eval run: the .dom.checkpoint appears alongside
the output, and the score/record/push path still works. The smoke-test artefacts
were removed rather than committed.

Refs homemaker-py-bk9.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB
This commit is contained in:
Claude 2026-09-06 20:58:05 +00:00
parent 58fe56eb55
commit 3ccbba7284
No known key found for this signature in database
1 changed files with 14 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
experiments/run_coldstart_baseline.py
View file

@ -122,15 +122,25 @@ def main() -> None:
formatter_class=argparse.RawDescriptionHelpFormatter) formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--budget", type=int, default=500000) ap.add_argument("--budget", type=int, default=500000)
ap.add_argument("--seeds", type=int, default=3) ap.add_argument("--seeds", type=int, default=3)
ap.add_argument("--slots", type=int, default=4) ap.add_argument("--slots", type=int, default=4,
help="concurrent runs; one worker each, so set it to your "
"core count (default 4)")
ap.add_argument("--checkpoint-every", type=int, default=None, metavar="N",
help="write each run's best-so-far .dom every N evals "
"(default: budget/20). The longest single run in the "
"first baseline took 62 h; without this, losing the "
"box at hour 61 loses all of it.")
ap.add_argument("--programmes", nargs="+", default=PROGRAMMES) ap.add_argument("--programmes", nargs="+", default=PROGRAMMES)
ap.add_argument("--dry-run", action="store_true") ap.add_argument("--dry-run", action="store_true")
args = ap.parse_args() args = ap.parse_args()
checkpoint_every = (args.checkpoint_every if args.checkpoint_every is not None
else max(1, args.budget // 20))
# seed-major: all programmes at seed 0, then seed 1, ... # seed-major: all programmes at seed 0, then seed 1, ...
queue = [(p, s) for s in range(args.seeds) for p in args.programmes] queue = [(p, s) for s in range(args.seeds) for p in args.programmes]
print(f"{len(queue)} runs, budget {args.budget}, {args.slots} slots, " print(f"{len(queue)} runs, budget {args.budget}, {args.slots} slots, "
f"seed-major order\n", flush=True) f"checkpoint every {checkpoint_every} evals, seed-major order\n",
flush=True)
if args.dry_run: if args.dry_run:
for p, s in queue: for p, s in queue:
print(f" would run {p} seed {s}") print(f" would run {p} seed {s}")
@ -146,7 +156,8 @@ def main() -> None:
fh = log.open("w") fh = log.open("w")
proc = subprocess.Popen( proc = subprocess.Popen(
["homemaker-evolve", "init.dom", "--budget", str(args.budget), ["homemaker-evolve", "init.dom", "--budget", str(args.budget),
"--seed", str(seed), "--workers", "1", "--output", str(out)], "--seed", str(seed), "--workers", "1", "--output", str(out),
"--checkpoint-every", str(checkpoint_every)],
cwd=d, stdout=subprocess.DEVNULL, stderr=fh) cwd=d, stdout=subprocess.DEVNULL, stderr=fh)
running[proc.pid] = (proc, prog, seed, out, fh, time.time()) running[proc.pid] = (proc, prog, seed, out, fh, time.time())
print(f" start {prog} seed {seed} -> {out.name}", flush=True) print(f" start {prog} seed {seed} -> {out.name}", flush=True)