diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 1509276..6611648 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -4,7 +4,7 @@ {"id":"homemaker-py-3y7","title":"Native fitness: adjacency/connectivity graph build + Merge_Divided semantics","description":"DESIGN.md §6 port scope, §7 Phase 3 (native fitness gates topology search at scale — §4.6). Port the door_width (1.2 m) adjacency graph (Urb Dom Graph), Merge_Divided, and the TWO-PHASE build: adjacency/level/vertical checks run on the UNMERGED tree, graphs rebuilt after Merge_Divided for storey processing (ProgrammeDriven.pm:83-103). Port faithfully — including has_vertical_connection's no-spatial-overlap stub (ProgrammeDriven.pm:399-423) unless the fidelity decision (§8.1) says otherwise; record the decision.","acceptance_criteria":"Graph edges/widths and merged structure match Perl on the 35-file corpus; vertical-connectivity fidelity decision recorded","status":"open","priority":1,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:38:23Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:38:23Z","dependency_count":0,"dependent_count":1,"comment_count":0} {"id":"homemaker-py-1p0","title":"Geometry inner loop: full-objective equal-offset ratio optimiser","description":"DESIGN.md §5.1, §7 Phase 1. Productionise experiments/optimize_fullfitness.py into homemaker: optimise(topology, x0=None) -\u003e (geometry, fitness). DOF = equal-offset division ratios of free branches (solver.free_branches, lowest-storey cut ownership), clipped to [eps, 1-eps]. Objective = full oracle fitness (never a proxy — §4.2 falsified). Must support warm-start x0 (§5.6) and a population/batch evaluation mode so each iteration scores via one batched oracle call (§4.6).","acceptance_criteria":"Reproduces or exceeds §4.5 gains (x1.24–x1.67, no new failures) on 2f45907, candidate-002, c964435; works as a library call on any corpus .dom","status":"open","priority":1,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:36:58Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:36:58Z","dependencies":[{"issue_id":"homemaker-py-1p0","depends_on_id":"homemaker-py-av5","type":"blocks","created_at":"2026-06-12T00:39:33Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":3,"comment_count":0} {"id":"homemaker-py-8cs","title":"Experiment: warm-vs-cold start of inner loop (Lamarckian inheritance)","description":"DESIGN.md §5.6, §4.6. Warm-starting a child topology's inner loop from the parent's optimised ratios is the main lever for cutting per-topology cost (~3 min/topology cold). Apply single topology mutations to optimised corpus designs, re-optimise warm (surviving cuts keep values, new cuts get heuristic defaults) vs cold, compare oracle-call counts to convergence at equal final fitness.","acceptance_criteria":"Speedup factor measured across \u003e=10 mutated topologies; decision recorded (expect order-of-magnitude; if \u003c2x, revisit §4.6 Phase-2 scoping)","status":"open","priority":1,"issue_type":"task","owner":"bruno@postle.net","created_at":"2026-06-11T23:36:58Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:36:58Z","dependencies":[{"issue_id":"homemaker-py-8cs","depends_on_id":"homemaker-py-1p0","type":"blocks","created_at":"2026-06-12T00:39:34Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} -{"id":"homemaker-py-av5","title":"Batched oracle: score many .dom files per invocation","description":"oracle.py currently scores one .dom per urb-fitness.pl call (~1.65 s/dom). DESIGN.md §4.6: batching amortises Perl startup to ~0.99 s/dom and is required so population/batch optimisers can score a whole generation in one oracle call. Extend oracle.py with a batch API: write N .dom files, one perl invocation, parse N .score/.fails pairs. Keep the single-file path for compatibility.","acceptance_criteria":"Batch of 35 corpus files scores in one perl invocation; per-file results identical to single-file calls; measured s/dom reported","status":"open","priority":1,"issue_type":"task","owner":"bruno@postle.net","created_at":"2026-06-11T23:36:56Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:36:56Z","dependency_count":0,"dependent_count":1,"comment_count":0} +{"id":"homemaker-py-av5","title":"Batched oracle: score many .dom files per invocation","description":"oracle.py currently scores one .dom per urb-fitness.pl call (~1.65 s/dom). DESIGN.md §4.6: batching amortises Perl startup to ~0.99 s/dom and is required so population/batch optimisers can score a whole generation in one oracle call. Extend oracle.py with a batch API: write N .dom files, one perl invocation, parse N .score/.fails pairs. Keep the single-file path for compatibility.","acceptance_criteria":"Batch of 35 corpus files scores in one perl invocation; per-file results identical to single-file calls; measured s/dom reported","status":"in_progress","priority":1,"issue_type":"task","assignee":"Bruno Postle","owner":"bruno@postle.net","created_at":"2026-06-11T23:36:56Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:50:40Z","started_at":"2026-06-11T23:50:40Z","dependency_count":0,"dependent_count":1,"comment_count":0} {"id":"homemaker-py-ccw","title":"Scaled topology search on native fitness","description":"DESIGN.md §7 Phase 3 closing step. Once native fitness passes corpus parity, re-run the Phase-2 memetic search at real scale (population/generations comparable to urb-evolve) on the native objective. This is the first point where the §1 scaling question gets a real answer.","acceptance_criteria":"Full-scale run on programme-house beats both urb-evolve and the small-scale Phase-2 result; larger programme attempted","status":"open","priority":2,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:38:59Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:38:59Z","dependencies":[{"issue_id":"homemaker-py-ccw","depends_on_id":"homemaker-py-uxz","type":"blocks","created_at":"2026-06-12T00:39:44Z","created_by":"Bruno Postle","metadata":"{}"},{"issue_id":"homemaker-py-ccw","depends_on_id":"homemaker-py-way","type":"blocks","created_at":"2026-06-12T00:39:45Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":2,"dependent_count":1,"comment_count":0} {"id":"homemaker-py-2g5","title":"Native fitness: occlusion/daylight/sun subsystem + crinkliness","description":"DESIGN.md §6 port scope — a whole subsystem, not a term. quality_daylight (Leaf.pm:281-296) needs Urb::Misc::Sun + Urb::Field::Occlusion (+CIESky); quality_uncrinkliness also takes the occlusion object. Indoor spaces return 1 for daylight; cost is outdoor spaces + crinkliness. Port Sun_horizontal (262980-minute normalisation) and the occlusion wall set from Dom-\u003eWalls.","acceptance_criteria":"Daylight and crinkliness factors match Perl (float tolerance) across the corpus, including multi-storey cases","status":"open","priority":2,"issue_type":"feature","owner":"bruno@postle.net","created_at":"2026-06-11T23:38:25Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:38:25Z","dependency_count":0,"dependent_count":1,"comment_count":0} {"id":"homemaker-py-way","title":"Benchmark: memetic loop vs urb-evolve at equal oracle-call budget (Phase 2 gate)","description":"DESIGN.md §7 Phase 2 gate. Compare against urb-evolve from the same seeds/programmes at equal oracle-evaluation budget — NOT generations (urb-evolve has diversity injection/culling baked in, so generations are not comparable). Go/no-go: memetic loop must beat equal-budget urb-evolve. Scaling up waits for native fitness.","acceptance_criteria":"Best-fitness and failure-count comparison at \u003e=2 budgets, \u003e=3 seeds; go/no-go decision recorded in DESIGN.md","status":"open","priority":2,"issue_type":"task","owner":"bruno@postle.net","created_at":"2026-06-11T23:37:28Z","created_by":"Bruno Postle","updated_at":"2026-06-11T23:37:28Z","dependencies":[{"issue_id":"homemaker-py-way","depends_on_id":"homemaker-py-b39","type":"blocks","created_at":"2026-06-12T00:39:39Z","created_by":"Bruno Postle","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} diff --git a/experiments/bench_batch_oracle.py b/experiments/bench_batch_oracle.py new file mode 100644 index 0000000..3ea4849 --- /dev/null +++ b/experiments/bench_batch_oracle.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Validate and benchmark the batched oracle (homemaker-py-av5). + +Copies the 35-file programme-house corpus into a scratch directory, scores +every file via single-file oracle calls and again via one batched invocation, +then checks the per-file fitness and failure sets are identical and reports +measured s/dom for both modes. +""" + +from __future__ import annotations + +import math +import shutil +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from homemaker import oracle # noqa: E402 + +URB = Path("/home/bruno/src/urb") +CORPUS = URB / "examples" / "programme-house" + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="bench_batch_") as tmp: + scratch = Path(tmp) + shutil.copy(CORPUS / "patterns.config", scratch) + doms = sorted(CORPUS.glob("*.dom")) + paths = [shutil.copy(d, scratch) for d in doms] + paths = [Path(p) for p in paths] + print(f"{len(paths)} corpus files -> {scratch}") + + t0 = time.perf_counter() + singles = [oracle.score(p, URB) for p in paths] + t_single = time.perf_counter() - t0 + + t0 = time.perf_counter() + batch = oracle.score_batch(paths, URB) + t_batch = time.perf_counter() - t0 + + # Urb's score is nondeterministic at the ~1 ULP level (Perl hash-order + # summation), so compare fitness with a tight relative tolerance; any + # real semantic difference (e.g. occlusion handling) would be far larger. + mismatches = 0 + for p, s, b in zip(paths, singles, batch): + if not math.isclose(s.fitness, b.fitness, rel_tol=1e-12) or s.fail_lines != b.fail_lines: + mismatches += 1 + print(f"MISMATCH {p.name}: single {s.fitness:.12g} ({s.n_fails} fails) " + f"vs batch {b.fitness:.12g} ({b.n_fails} fails)") + print(f" single-only: {sorted(set(s.fail_lines) - set(b.fail_lines))}") + print(f" batch-only: {sorted(set(b.fail_lines) - set(s.fail_lines))}") + + n = len(paths) + print(f"\nsingle-file: {t_single:.2f} s total, {t_single / n:.3f} s/dom") + print(f"batched: {t_batch:.2f} s total, {t_batch / n:.3f} s/dom") + print(f"speedup: x{t_single / t_batch:.2f}") + if mismatches: + print(f"\nFAIL: {mismatches}/{n} files differ between single and batch") + return 1 + print(f"\nOK: all {n} files identical (fitness to 1e-12 rel, exact failure set) in both modes") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/homemaker/oracle.py b/src/homemaker/oracle.py index 4c6fe65..8f86e2d 100644 --- a/src/homemaker/oracle.py +++ b/src/homemaker/oracle.py @@ -1,9 +1,22 @@ -"""Phase-1 fitness oracle: score a ``.dom`` via Urb's ``urb-fitness.pl``. +"""Phase-1 fitness oracle: score ``.dom`` files via Urb's ``urb-fitness.pl``. This is the only throwaway component. It shells out to the Perl evaluator so we can validate the Python search core against the trusted fitness before porting -fitness to Python (Phase 2). ``urb-fitness.pl`` reads ``patterns.config`` from +fitness to Python (Phase 3). ``urb-fitness.pl`` reads ``patterns.config`` from its working directory, so the ``.dom`` must live beside the programme config. + +``urb-fitness.pl`` accepts many ``.dom`` paths per invocation; ``score_batch`` +exploits this so the ~0.65 s Perl startup amortises across a generation +(DESIGN.md §4.6: ~0.99 s/dom batched vs ~1.65 s/dom single). Note the Perl +script computes the occlusion field from the *first* dom in a batch and reuses +it for the rest; ``experiments/bench_batch_oracle.py`` verifies this leaves +corpus scores identical to single-file calls. + +Two flavours of Urb-side nondeterminism to know about (both from Perl's +per-process hash-order randomisation, neither a batching artifact): ``.fails`` +line *order* varies between runs (use ``Score.fail_lines``), and the score +itself can flip by ~1 ULP. Compare fitness with a relative tolerance +(``math.isclose(..., rel_tol=1e-12)``), never ``==``. """ from __future__ import annotations @@ -12,6 +25,7 @@ import os import subprocess from dataclasses import dataclass from pathlib import Path +from typing import Sequence DEFAULT_URB_ROOT = Path("/home/bruno/src/urb") @@ -21,32 +35,66 @@ class Score: fitness: float fails: str # raw .fails content (YAML and/or plain lines) + @property + def fail_lines(self) -> tuple[str, ...]: + """Failure messages as a sorted tuple — Perl's per-process hash-order + randomisation shuffles the raw ``.fails`` line order between runs, so + comparisons must be order-insensitive.""" + return tuple( + sorted(line.strip() for line in self.fails.splitlines() if line.strip() and line.strip() != "---") + ) + @property def n_fails(self) -> int: - return sum(1 for line in self.fails.splitlines() if line.strip() and line.strip() != "---") + return len(self.fail_lines) -def score(dom_path: str | Path, urb_root: str | Path = DEFAULT_URB_ROOT) -> Score: - dom_path = Path(dom_path).resolve() +def score_batch( + dom_paths: Sequence[str | Path], urb_root: str | Path = DEFAULT_URB_ROOT +) -> list[Score]: + """Score many ``.dom`` files in one ``urb-fitness.pl`` invocation. + + All files must live in the same directory (the working directory, where + ``patterns.config`` is found). Results are returned in input order. + """ + paths = [Path(p).resolve() for p in dom_paths] + if not paths: + return [] + cwd = paths[0].parent + for p in paths: + if p.parent != cwd: + raise ValueError(f"batch spans directories: {p} not in {cwd}") + Path(f"{p}.score").unlink(missing_ok=True) + Path(f"{p}.fails").unlink(missing_ok=True) + urb_root = Path(urb_root).resolve() - score_file = Path(f"{dom_path}.score") - fails_file = Path(f"{dom_path}.fails") - for f in (score_file, fails_file): - f.unlink(missing_ok=True) - env = {**os.environ, "DEBUG": "1"} proc = subprocess.run( - ["perl", f"-I{urb_root}/lib", str(urb_root / "bin" / "urb-fitness.pl"), dom_path.name], - cwd=dom_path.parent, + ["perl", f"-I{urb_root}/lib", str(urb_root / "bin" / "urb-fitness.pl")] + + [p.name for p in paths], + cwd=cwd, env=env, capture_output=True, text=True, ) - if not score_file.exists(): - raise RuntimeError( - f"urb-fitness.pl produced no score for {dom_path}\n" - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + + results = [] + for p in paths: + score_file = Path(f"{p}.score") + if not score_file.exists(): + raise RuntimeError( + f"urb-fitness.pl produced no score for {p}\n" + f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + fails_file = Path(f"{p}.fails") + results.append( + Score( + fitness=float(score_file.read_text().strip()), + fails=fails_file.read_text() if fails_file.exists() else "", + ) ) - fitness = float(score_file.read_text().strip()) - fails = fails_file.read_text() if fails_file.exists() else "" - return Score(fitness=fitness, fails=fails) + return results + + +def score(dom_path: str | Path, urb_root: str | Path = DEFAULT_URB_ROOT) -> Score: + return score_batch([dom_path], urb_root)[0]