diff --git a/examples/programme-house/coldstart-1000-s0.dom b/examples/programme-house/coldstart-1000-s0.dom new file mode 100644 index 0000000..d02c11c --- /dev/null +++ b/examples/programme-house/coldstart-1000-s0.dom @@ -0,0 +1,96 @@ +node: +- - 1.67100188077385 + - 11.9169190395339 +- - 8.72124918939699 + - 13.7119442286569 +- - 6.79637028658874 + - 21.3797854212389 +- - -0.42152047183069 + - 19.9368564494913 +perimeter: + a: private + b: null + c: null + d: null +rotation: 1 +division: +- 0.47261205221974323 +- 0.47261205221974323 +height: 3.0 +elevation: 0.0 +wall_inner: 0.08 +wall_outer: 0.25 +l: + rotation: 1 + division: + - 0.5340952149109011 + - 0.5340952149109011 + l: + type: b2 + rotation: 0 + r: + rotation: 1 + division: + - 0.5657418003155832 + - 0.5657418003155832 + l: + type: C + rotation: 0 + r: + type: t2 + rotation: 0 +r: + rotation: 1 + division: + - 0.7100729116945828 + - 0.7100729116945828 + l: + rotation: 0 + division: + - 0.34202939905806784 + - 0.34202939905806784 + l: + type: O + rotation: 0 + r: + type: l1 + rotation: 0 + r: + rotation: 1 + division: + - 0.5095335299823702 + - 0.5095335299823702 + l: + type: t3 + rotation: 0 + r: + type: C + rotation: 0 +above: + rotation: 0 + division: + - 0.61355312064873 + - 0.61355312064873 + height: 3.0 + l: + rotation: 0 + division: + - 0.5 + - 0.5 + l: + type: C + rotation: 0 + r: + type: O + rotation: 0 + r: + rotation: 0 + division: + - 0.8 + - 0.8 + l: + type: b1 + rotation: 0 + r: + type: t1 + rotation: 0 diff --git a/examples/programme-house/coldstart-1000-s0.log b/examples/programme-house/coldstart-1000-s0.log new file mode 100644 index 0000000..52c0322 --- /dev/null +++ b/examples/programme-house/coldstart-1000-s0.log @@ -0,0 +1,44 @@ +seed : /home/user/homemaker-layout/examples/programme-house/init.dom +programme : programme-house +budget : 1000 +pop : 16 +child_budget : 80 +workers : 1 +rng seed : 0 +leaf sharing : True (factor=3) +superpose : False +multi_use : False +conn grade : False +use tiers : False +bridge circulation : False +ruin recreate : False +collapse in-search : True +shapecurve warmstart : False +shapecurve prune : False +output : /home/user/homemaker-layout/examples/programme-house/coldstart-1000-s0.dom +[ 80 evals] best 9.96032e-11 (fails 17) via construct/0 +[ 160 evals] best 5.18243e-09 (fails 16) via construct/1 +[ 320 evals] best 9.65534e-06 (fails 7) via construct/3 + +--- finishing (homemaker-py-3l6): unfold + polish 500 evals --- +[finish] unfold: materialised 0 shared-leaf copies +[ 200 evals] best 1.97223e-05 (fails 7) via seed +[ 280 evals] best 1.99855e-05 (fails 7) via place_missing noop +[ 440 evals] best 2.02918e-05 (fails 7) via place_missing noop + +--- collapse (homemaker-py-94g): finish-time cell→room relabel --- +[finish] collapse: 10 → 7 fails (applied) + +--- done --- +elapsed : 6.1s +evals : 1800 across 21 topologies +best : 2.02918e-05 (7 fails) via place_missing noop+collapse + +improvement history: + [ 80] 9.96032e-11 (share:construct/0) + [ 160] 5.18243e-09 (share:construct/1) + [ 320] 9.65534e-06 (share:construct/3) + [ 1480] 1.97223e-05 (polish:seed) + [ 1560] 1.99855e-05 (polish:place_missing noop) + [ 1720] 2.02918e-05 (polish:place_missing noop) +written : /home/user/homemaker-layout/examples/programme-house/coldstart-1000-s0.dom diff --git a/experiments/results/coldstart_baseline.tsv b/experiments/results/coldstart_baseline.tsv new file mode 100644 index 0000000..5eee2e7 --- /dev/null +++ b/experiments/results/coldstart_baseline.tsv @@ -0,0 +1,2 @@ +programme seed budget fails hard soft score elapsed_s dom +programme-house 0 1000 7 3 4 2.02918e-05 10.0 coldstart-1000-s0.dom diff --git a/experiments/run_coldstart_baseline.py b/experiments/run_coldstart_baseline.py new file mode 100644 index 0000000..29096c5 --- /dev/null +++ b/experiments/run_coldstart_baseline.py @@ -0,0 +1,167 @@ +"""Cold-start re-baseline of the four example programmes (`homemaker-py-ssz` fallout). + +The objective changed (DESIGN.md §38.10/§38.11: crinkliness is declared per +space), so every prior corpus fail count is stale. This re-runs all four +programmes cold, from `init.dom`, at a fixed budget. + +Design notes, all of which matter for the result being trustworthy: + +* **One worker per run, four runs at a time.** The box has 4 cores. Running + four single-worker searches beats one four-worker search here: it saturates + the cores just as well AND avoids `homemaker-py-b8g`, the parallel/BLAS + non-determinism that makes `n_workers>1` runs irreproducible. A baseline + nobody can reproduce is not a baseline. +* **Seed-major order.** The queue runs seed 0 of every programme, then seed 1, + then seed 2 -- so if the box is lost half way we have all four programmes at + fewer seeds, rather than one programme at three seeds and nothing else. +* **Commit and push after every finished run.** This is an ephemeral container; + it is reclaimed on inactivity or session end. Anything not pushed is gone. Git + calls are serialised under a lock file so the runner cannot race a human (or + another agent) committing in the same tree. +* **Scored by the shipped scorer**, from inside the programme directory, exactly + as CLAUDE.md requires -- `homemaker-fitness` resolves patterns.config and + writes .score/.fails relative to cwd. + +Usage:: + + python experiments/run_coldstart_baseline.py --budget 500000 --seeds 3 + python experiments/run_coldstart_baseline.py --budget 2000 --seeds 1 --dry-run +""" + +from __future__ import annotations + +import argparse +import csv +import fcntl +import subprocess +import time +from contextlib import contextmanager +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +PROGRAMMES = ["harbor-house", "maple-court", "health-centre", "programme-house"] +LOCK = REPO / ".git" / "coldstart-git.lock" +RESULTS = REPO / "experiments" / "results" / "coldstart_baseline.tsv" +FIELDS = ["programme", "seed", "budget", "fails", "hard", "soft", "score", + "elapsed_s", "dom"] + + +@contextmanager +def git_lock(): + LOCK.parent.mkdir(parents=True, exist_ok=True) + with open(LOCK, "w") as fh: + fcntl.flock(fh, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(fh, fcntl.LOCK_UN) + + +def score(dom_path: Path) -> tuple[int, int, int, float]: + """(fails, hard, soft, score) via the shipped scorer, run from the + programme dir as CLAUDE.md requires.""" + from homemaker_layout.fitness import classify_fail_tier + subprocess.run(["homemaker-fitness", dom_path.name], + cwd=dom_path.parent, capture_output=True, text=True) + fails_file = dom_path.with_suffix(".dom.fails") + lines = [ln.strip() for ln in fails_file.read_text().splitlines() + if ln.strip()] if fails_file.exists() else [] + hard = sum(1 for ln in lines if classify_fail_tier(ln) == "hard") + score_file = dom_path.with_suffix(".dom.score") + val = float(score_file.read_text().strip()) if score_file.exists() else float("nan") + return len(lines), hard, len(lines) - hard, val + + +def record_and_push(row: dict) -> None: + rows = [] + if RESULTS.exists(): + rows = list(csv.DictReader(RESULTS.open(), delimiter="\t")) + rows.append({k: str(row[k]) for k in FIELDS}) + RESULTS.parent.mkdir(parents=True, exist_ok=True) + with RESULTS.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=FIELDS, delimiter="\t") + w.writeheader() + w.writerows(rows) + + msg = (f"coldstart {row['programme']} seed {row['seed']} @ {row['budget']}: " + f"{row['fails']} fails ({row['hard']}h/{row['soft']}s)") + with git_lock(): + subprocess.run(["git", "add", "-A"], cwd=REPO, capture_output=True) + subprocess.run( + ["git", "commit", "-q", "-m", msg + "\n\n" + "Cold-start re-baseline after the DESIGN.md 38.10/38.11 objective\n" + "change. Single worker (avoids homemaker-py-b8g), scored by the\n" + "shipped scorer from the programme directory.\n\n" + "Co-Authored-By: Claude Opus 5 \n" + "Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB"], + cwd=REPO, capture_output=True) + for attempt in range(4): + subprocess.run(["git", "pull", "--rebase", "-q", "origin", + "claude/beads-project-intro-fjiez3"], + cwd=REPO, capture_output=True) + p = subprocess.run(["git", "push", "-q", "origin", + "claude/beads-project-intro-fjiez3"], + cwd=REPO, capture_output=True) + if p.returncode == 0: + break + time.sleep(2 ** (attempt + 1)) + print(f" pushed: {msg}", flush=True) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--budget", type=int, default=500000) + ap.add_argument("--seeds", type=int, default=3) + ap.add_argument("--slots", type=int, default=4) + ap.add_argument("--programmes", nargs="+", default=PROGRAMMES) + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + # seed-major: all programmes at seed 0, then seed 1, ... + queue = [(p, s) for s in range(args.seeds) for p in args.programmes] + print(f"{len(queue)} runs, budget {args.budget}, {args.slots} slots, " + f"seed-major order\n", flush=True) + if args.dry_run: + for p, s in queue: + print(f" would run {p} seed {s}") + return + + running: dict = {} + while queue or running: + while queue and len(running) < args.slots: + prog, seed = queue.pop(0) + d = REPO / "examples" / prog + out = d / f"coldstart-{args.budget}-s{seed}.dom" + log = d / f"coldstart-{args.budget}-s{seed}.log" + fh = log.open("w") + proc = subprocess.Popen( + ["homemaker-evolve", "init.dom", "--budget", str(args.budget), + "--seed", str(seed), "--workers", "1", "--output", str(out)], + cwd=d, stdout=subprocess.DEVNULL, stderr=fh) + running[proc.pid] = (proc, prog, seed, out, fh, time.time()) + print(f" start {prog} seed {seed} -> {out.name}", flush=True) + + time.sleep(10) + for pid, (proc, prog, seed, out, fh, t0) in list(running.items()): + if proc.poll() is None: + continue + fh.close() + del running[pid] + elapsed = round(time.time() - t0, 1) + if not out.exists(): + print(f" FAILED {prog} seed {seed} (rc={proc.returncode}, " + f"{elapsed}s) -- see the .log", flush=True) + continue + n, hard, soft, val = score(out) + print(f" done {prog} seed {seed}: {n} fails " + f"({hard}h/{soft}s) score {val:.4g} in {elapsed}s", flush=True) + record_and_push(dict(programme=prog, seed=seed, budget=args.budget, + fails=n, hard=hard, soft=soft, score=f"{val:.6g}", + elapsed_s=elapsed, dom=out.name)) + + print("\n=== all runs complete ===", flush=True) + + +if __name__ == "__main__": + main()