coldstart programme-house seed 0 @ 1000: 7 fails (3h/4s)
Cold-start re-baseline after the DESIGN.md 38.10/38.11 objective change. Single worker (avoids homemaker-py-b8g), scored by the shipped scorer from the programme directory. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB
This commit is contained in:
parent
45eff37f96
commit
e2d5838433
4 changed files with 309 additions and 0 deletions
96
examples/programme-house/coldstart-1000-s0.dom
Normal file
96
examples/programme-house/coldstart-1000-s0.dom
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
node:
|
||||||
|
- - 1.67100188077385
|
||||||
|
- 11.9169190395339
|
||||||
|
- - 8.72124918939699
|
||||||
|
- 13.7119442286569
|
||||||
|
- - 6.79637028658874
|
||||||
|
- 21.3797854212389
|
||||||
|
- - -0.42152047183069
|
||||||
|
- 19.9368564494913
|
||||||
|
perimeter:
|
||||||
|
a: private
|
||||||
|
b: null
|
||||||
|
c: null
|
||||||
|
d: null
|
||||||
|
rotation: 1
|
||||||
|
division:
|
||||||
|
- 0.47261205221974323
|
||||||
|
- 0.47261205221974323
|
||||||
|
height: 3.0
|
||||||
|
elevation: 0.0
|
||||||
|
wall_inner: 0.08
|
||||||
|
wall_outer: 0.25
|
||||||
|
l:
|
||||||
|
rotation: 1
|
||||||
|
division:
|
||||||
|
- 0.5340952149109011
|
||||||
|
- 0.5340952149109011
|
||||||
|
l:
|
||||||
|
type: b2
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
rotation: 1
|
||||||
|
division:
|
||||||
|
- 0.5657418003155832
|
||||||
|
- 0.5657418003155832
|
||||||
|
l:
|
||||||
|
type: C
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
type: t2
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
rotation: 1
|
||||||
|
division:
|
||||||
|
- 0.7100729116945828
|
||||||
|
- 0.7100729116945828
|
||||||
|
l:
|
||||||
|
rotation: 0
|
||||||
|
division:
|
||||||
|
- 0.34202939905806784
|
||||||
|
- 0.34202939905806784
|
||||||
|
l:
|
||||||
|
type: O
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
type: l1
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
rotation: 1
|
||||||
|
division:
|
||||||
|
- 0.5095335299823702
|
||||||
|
- 0.5095335299823702
|
||||||
|
l:
|
||||||
|
type: t3
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
type: C
|
||||||
|
rotation: 0
|
||||||
|
above:
|
||||||
|
rotation: 0
|
||||||
|
division:
|
||||||
|
- 0.61355312064873
|
||||||
|
- 0.61355312064873
|
||||||
|
height: 3.0
|
||||||
|
l:
|
||||||
|
rotation: 0
|
||||||
|
division:
|
||||||
|
- 0.5
|
||||||
|
- 0.5
|
||||||
|
l:
|
||||||
|
type: C
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
type: O
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
rotation: 0
|
||||||
|
division:
|
||||||
|
- 0.8
|
||||||
|
- 0.8
|
||||||
|
l:
|
||||||
|
type: b1
|
||||||
|
rotation: 0
|
||||||
|
r:
|
||||||
|
type: t1
|
||||||
|
rotation: 0
|
||||||
44
examples/programme-house/coldstart-1000-s0.log
Normal file
44
examples/programme-house/coldstart-1000-s0.log
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
seed : /home/user/homemaker-layout/examples/programme-house/init.dom
|
||||||
|
programme : programme-house
|
||||||
|
budget : 1000
|
||||||
|
pop : 16
|
||||||
|
child_budget : 80
|
||||||
|
workers : 1
|
||||||
|
rng seed : 0
|
||||||
|
leaf sharing : True (factor=3)
|
||||||
|
superpose : False
|
||||||
|
multi_use : False
|
||||||
|
conn grade : False
|
||||||
|
use tiers : False
|
||||||
|
bridge circulation : False
|
||||||
|
ruin recreate : False
|
||||||
|
collapse in-search : True
|
||||||
|
shapecurve warmstart : False
|
||||||
|
shapecurve prune : False
|
||||||
|
output : /home/user/homemaker-layout/examples/programme-house/coldstart-1000-s0.dom
|
||||||
|
[ 80 evals] best 9.96032e-11 (fails 17) via construct/0
|
||||||
|
[ 160 evals] best 5.18243e-09 (fails 16) via construct/1
|
||||||
|
[ 320 evals] best 9.65534e-06 (fails 7) via construct/3
|
||||||
|
|
||||||
|
--- finishing (homemaker-py-3l6): unfold + polish 500 evals ---
|
||||||
|
[finish] unfold: materialised 0 shared-leaf copies
|
||||||
|
[ 200 evals] best 1.97223e-05 (fails 7) via seed
|
||||||
|
[ 280 evals] best 1.99855e-05 (fails 7) via place_missing noop
|
||||||
|
[ 440 evals] best 2.02918e-05 (fails 7) via place_missing noop
|
||||||
|
|
||||||
|
--- collapse (homemaker-py-94g): finish-time cell→room relabel ---
|
||||||
|
[finish] collapse: 10 → 7 fails (applied)
|
||||||
|
|
||||||
|
--- done ---
|
||||||
|
elapsed : 6.1s
|
||||||
|
evals : 1800 across 21 topologies
|
||||||
|
best : 2.02918e-05 (7 fails) via place_missing noop+collapse
|
||||||
|
|
||||||
|
improvement history:
|
||||||
|
[ 80] 9.96032e-11 (share:construct/0)
|
||||||
|
[ 160] 5.18243e-09 (share:construct/1)
|
||||||
|
[ 320] 9.65534e-06 (share:construct/3)
|
||||||
|
[ 1480] 1.97223e-05 (polish:seed)
|
||||||
|
[ 1560] 1.99855e-05 (polish:place_missing noop)
|
||||||
|
[ 1720] 2.02918e-05 (polish:place_missing noop)
|
||||||
|
written : /home/user/homemaker-layout/examples/programme-house/coldstart-1000-s0.dom
|
||||||
2
experiments/results/coldstart_baseline.tsv
Normal file
2
experiments/results/coldstart_baseline.tsv
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
programme seed budget fails hard soft score elapsed_s dom
|
||||||
|
programme-house 0 1000 7 3 4 2.02918e-05 10.0 coldstart-1000-s0.dom
|
||||||
|
167
experiments/run_coldstart_baseline.py
Normal file
167
experiments/run_coldstart_baseline.py
Normal file
|
|
@ -0,0 +1,167 @@
|
||||||
|
"""Cold-start re-baseline of the four example programmes (`homemaker-py-ssz` fallout).
|
||||||
|
|
||||||
|
The objective changed (DESIGN.md §38.10/§38.11: crinkliness is declared per
|
||||||
|
space), so every prior corpus fail count is stale. This re-runs all four
|
||||||
|
programmes cold, from `init.dom`, at a fixed budget.
|
||||||
|
|
||||||
|
Design notes, all of which matter for the result being trustworthy:
|
||||||
|
|
||||||
|
* **One worker per run, four runs at a time.** The box has 4 cores. Running
|
||||||
|
four single-worker searches beats one four-worker search here: it saturates
|
||||||
|
the cores just as well AND avoids `homemaker-py-b8g`, the parallel/BLAS
|
||||||
|
non-determinism that makes `n_workers>1` runs irreproducible. A baseline
|
||||||
|
nobody can reproduce is not a baseline.
|
||||||
|
* **Seed-major order.** The queue runs seed 0 of every programme, then seed 1,
|
||||||
|
then seed 2 -- so if the box is lost half way we have all four programmes at
|
||||||
|
fewer seeds, rather than one programme at three seeds and nothing else.
|
||||||
|
* **Commit and push after every finished run.** This is an ephemeral container;
|
||||||
|
it is reclaimed on inactivity or session end. Anything not pushed is gone. Git
|
||||||
|
calls are serialised under a lock file so the runner cannot race a human (or
|
||||||
|
another agent) committing in the same tree.
|
||||||
|
* **Scored by the shipped scorer**, from inside the programme directory, exactly
|
||||||
|
as CLAUDE.md requires -- `homemaker-fitness` resolves patterns.config and
|
||||||
|
writes .score/.fails relative to cwd.
|
||||||
|
|
||||||
|
Usage::
|
||||||
|
|
||||||
|
python experiments/run_coldstart_baseline.py --budget 500000 --seeds 3
|
||||||
|
python experiments/run_coldstart_baseline.py --budget 2000 --seeds 1 --dry-run
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import fcntl
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parent.parent
|
||||||
|
PROGRAMMES = ["harbor-house", "maple-court", "health-centre", "programme-house"]
|
||||||
|
LOCK = REPO / ".git" / "coldstart-git.lock"
|
||||||
|
RESULTS = REPO / "experiments" / "results" / "coldstart_baseline.tsv"
|
||||||
|
FIELDS = ["programme", "seed", "budget", "fails", "hard", "soft", "score",
|
||||||
|
"elapsed_s", "dom"]
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def git_lock():
|
||||||
|
LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(LOCK, "w") as fh:
|
||||||
|
fcntl.flock(fh, fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
fcntl.flock(fh, fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
def score(dom_path: Path) -> tuple[int, int, int, float]:
|
||||||
|
"""(fails, hard, soft, score) via the shipped scorer, run from the
|
||||||
|
programme dir as CLAUDE.md requires."""
|
||||||
|
from homemaker_layout.fitness import classify_fail_tier
|
||||||
|
subprocess.run(["homemaker-fitness", dom_path.name],
|
||||||
|
cwd=dom_path.parent, capture_output=True, text=True)
|
||||||
|
fails_file = dom_path.with_suffix(".dom.fails")
|
||||||
|
lines = [ln.strip() for ln in fails_file.read_text().splitlines()
|
||||||
|
if ln.strip()] if fails_file.exists() else []
|
||||||
|
hard = sum(1 for ln in lines if classify_fail_tier(ln) == "hard")
|
||||||
|
score_file = dom_path.with_suffix(".dom.score")
|
||||||
|
val = float(score_file.read_text().strip()) if score_file.exists() else float("nan")
|
||||||
|
return len(lines), hard, len(lines) - hard, val
|
||||||
|
|
||||||
|
|
||||||
|
def record_and_push(row: dict) -> None:
|
||||||
|
rows = []
|
||||||
|
if RESULTS.exists():
|
||||||
|
rows = list(csv.DictReader(RESULTS.open(), delimiter="\t"))
|
||||||
|
rows.append({k: str(row[k]) for k in FIELDS})
|
||||||
|
RESULTS.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with RESULTS.open("w", newline="") as fh:
|
||||||
|
w = csv.DictWriter(fh, fieldnames=FIELDS, delimiter="\t")
|
||||||
|
w.writeheader()
|
||||||
|
w.writerows(rows)
|
||||||
|
|
||||||
|
msg = (f"coldstart {row['programme']} seed {row['seed']} @ {row['budget']}: "
|
||||||
|
f"{row['fails']} fails ({row['hard']}h/{row['soft']}s)")
|
||||||
|
with git_lock():
|
||||||
|
subprocess.run(["git", "add", "-A"], cwd=REPO, capture_output=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "commit", "-q", "-m", msg + "\n\n"
|
||||||
|
"Cold-start re-baseline after the DESIGN.md 38.10/38.11 objective\n"
|
||||||
|
"change. Single worker (avoids homemaker-py-b8g), scored by the\n"
|
||||||
|
"shipped scorer from the programme directory.\n\n"
|
||||||
|
"Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>\n"
|
||||||
|
"Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB"],
|
||||||
|
cwd=REPO, capture_output=True)
|
||||||
|
for attempt in range(4):
|
||||||
|
subprocess.run(["git", "pull", "--rebase", "-q", "origin",
|
||||||
|
"claude/beads-project-intro-fjiez3"],
|
||||||
|
cwd=REPO, capture_output=True)
|
||||||
|
p = subprocess.run(["git", "push", "-q", "origin",
|
||||||
|
"claude/beads-project-intro-fjiez3"],
|
||||||
|
cwd=REPO, capture_output=True)
|
||||||
|
if p.returncode == 0:
|
||||||
|
break
|
||||||
|
time.sleep(2 ** (attempt + 1))
|
||||||
|
print(f" pushed: {msg}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__,
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
ap.add_argument("--budget", type=int, default=500000)
|
||||||
|
ap.add_argument("--seeds", type=int, default=3)
|
||||||
|
ap.add_argument("--slots", type=int, default=4)
|
||||||
|
ap.add_argument("--programmes", nargs="+", default=PROGRAMMES)
|
||||||
|
ap.add_argument("--dry-run", action="store_true")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
# seed-major: all programmes at seed 0, then seed 1, ...
|
||||||
|
queue = [(p, s) for s in range(args.seeds) for p in args.programmes]
|
||||||
|
print(f"{len(queue)} runs, budget {args.budget}, {args.slots} slots, "
|
||||||
|
f"seed-major order\n", flush=True)
|
||||||
|
if args.dry_run:
|
||||||
|
for p, s in queue:
|
||||||
|
print(f" would run {p} seed {s}")
|
||||||
|
return
|
||||||
|
|
||||||
|
running: dict = {}
|
||||||
|
while queue or running:
|
||||||
|
while queue and len(running) < args.slots:
|
||||||
|
prog, seed = queue.pop(0)
|
||||||
|
d = REPO / "examples" / prog
|
||||||
|
out = d / f"coldstart-{args.budget}-s{seed}.dom"
|
||||||
|
log = d / f"coldstart-{args.budget}-s{seed}.log"
|
||||||
|
fh = log.open("w")
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
["homemaker-evolve", "init.dom", "--budget", str(args.budget),
|
||||||
|
"--seed", str(seed), "--workers", "1", "--output", str(out)],
|
||||||
|
cwd=d, stdout=subprocess.DEVNULL, stderr=fh)
|
||||||
|
running[proc.pid] = (proc, prog, seed, out, fh, time.time())
|
||||||
|
print(f" start {prog} seed {seed} -> {out.name}", flush=True)
|
||||||
|
|
||||||
|
time.sleep(10)
|
||||||
|
for pid, (proc, prog, seed, out, fh, t0) in list(running.items()):
|
||||||
|
if proc.poll() is None:
|
||||||
|
continue
|
||||||
|
fh.close()
|
||||||
|
del running[pid]
|
||||||
|
elapsed = round(time.time() - t0, 1)
|
||||||
|
if not out.exists():
|
||||||
|
print(f" FAILED {prog} seed {seed} (rc={proc.returncode}, "
|
||||||
|
f"{elapsed}s) -- see the .log", flush=True)
|
||||||
|
continue
|
||||||
|
n, hard, soft, val = score(out)
|
||||||
|
print(f" done {prog} seed {seed}: {n} fails "
|
||||||
|
f"({hard}h/{soft}s) score {val:.4g} in {elapsed}s", flush=True)
|
||||||
|
record_and_push(dict(programme=prog, seed=seed, budget=args.budget,
|
||||||
|
fails=n, hard=hard, soft=soft, score=f"{val:.6g}",
|
||||||
|
elapsed_s=elapsed, dom=out.name))
|
||||||
|
|
||||||
|
print("\n=== all runs complete ===", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Loading…
Add table
Reference in a new issue