bruno/homemaker-layout
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

homemaker-py-2g7.3: hard/soft fail tiering behind --use-tiers flag

Browse source 
Splits the flat outer-search comparator (-n_fails, fitness) into a tiered
(-n_hard, -n_soft, fitness) so search budget stops being spent polishing
SOFT shape fails (crinkliness/proportion/size/width/edge-too-long/
staircase-volume) while HARD structural fails (missing space, wrong/
required level, level/circulation/vertical connectivity, adjacency,
stairs, covered-outside, storey limits, public access) remain unfixed.

fitness.classify_fail_tier/tier_counts classify every fail string emitted
across fitness.py and graph.py, raising on anything unrecognised so new
fail sites must declare a tier. Validated against all real fail strings in
the checked-in corpus plus every fail-emission call site read from source.

driver.Individual gains n_hard/n_soft (populated from innerloop.Result.
fail_lines); search(use_tiers=...) swaps the comparator when set (default
off, so existing runs are unaffected — inner-loop 0.5^n cliff untouched).
evolve.py exposes --use-tiers / HOMEMAKER_USE_TIERS.

experiments/tier_ab_2g7_3.py runs the acceptance A/B (harbor+maple, 3
seeds, 20k evals) in the background; results pending.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LSwQwpEaHFBkeVSDDWd75S
This commit is contained in:
Bruno Postle 2026-08-02 16:00:39 +01:00
parent 91ff4fdcaa
commit 8efdc02fd9
7 changed files with 387 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
.beads/issues.jsonl
View file

File diff suppressed because one or more lines are too long

106
experiments/tier_ab_2g7_3.py Normal file
View file

@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Hard/soft fail tiering A/B (homemaker-py-2g7.3, DESIGN.md §37).
Acceptance criteria: "tiered comparator behind a flag with A/B on harbor+maple
(3 seeds, 20k evals): hard-fail count at budget strictly better or equal on
mean, no §4.9 regression; report shows hard/soft split."
Compares the outer comparator (-n_fails, fitness) [use_tiers=False, the
existing default] against (-n_hard, -n_soft, fitness) [use_tiers=True] on
harbor-house and maple-court, 3 seeds each, budget=20000 native evals/run.
Reports mean hard/soft/total fail counts per config and the per-seed deltas.
Usage:
URB_NO_OCCLUSION=1 python3 experiments/tier_ab_2g7_3.py \
[budget] [n_seeds] [workers] [out_dir]
Defaults: budget=20000, n_seeds=3, workers=4, scratch/tier_ab_2g7_3.
"""
from __future__ import annotations
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from homemaker_layout import dom, driver # noqa: E402
REPO = Path(__file__).resolve().parents[1]
PROGRAMMES = ["harbor-house", "maple-court"]
def _run(programme_dir: Path, seed: int, budget: int, workers: int, use_tiers: bool):
seed_root = dom.load(str(programme_dir / "init.dom"))
t0 = time.perf_counter()
r = driver.search(
seed_root, programme_dir, budget=budget, pop_size=16, child_budget=80,
seed_budget=300, p_crossover=0.2, seed=seed, n_workers=workers,
leaf_sharing=True, use_tiers=use_tiers,
)
dt = time.perf_counter() - t0
return {
"n_fails": r.best.n_fails, "n_hard": r.best.n_hard, "n_soft": r.best.n_soft,
"fitness": r.best.fitness, "n_evals": r.n_evals, "wall_s": dt,
}
def main() -> int:
budget = int(sys.argv[1]) if len(sys.argv) > 1 else 20000
n_seeds = int(sys.argv[2]) if len(sys.argv) > 2 else 3
workers = int(sys.argv[3]) if len(sys.argv) > 3 else 4
out_dir = Path(sys.argv[4]) if len(sys.argv) > 4 else (REPO / "scratch" / "tier_ab_2g7_3")
out_dir.mkdir(parents=True, exist_ok=True)
print(f"budget : {budget}")
print(f"n_seeds : {n_seeds}")
print(f"workers : {workers}")
print(f"programmes: {PROGRAMMES}")
print(flush=True)
t_start = time.perf_counter()
results: dict[str, dict[str, list[dict]]] = {}
for prog_name in PROGRAMMES:
programme_dir = REPO / "examples" / prog_name
results[prog_name] = {"flat": [], "tiered": []}
print(f"=== {prog_name} ===", flush=True)
for seed in range(n_seeds):
for label, use_tiers in (("flat", False), ("tiered", True)):
res = _run(programme_dir, seed, budget, workers, use_tiers)
results[prog_name][label].append(res)
print(f" seed {seed} {label:6s}: hard={res['n_hard']} "
f"soft={res['n_soft']} total={res['n_fails']} "
f"fitness={res['fitness']:.6g} evals={res['n_evals']} "
f"({res['wall_s']:.0f}s)", flush=True)
print()
print("=" * 72)
print("SUMMARY (mean over seeds)")
print("=" * 72)
overall_ok = True
for prog_name in PROGRAMMES:
for label in ("flat", "tiered"):
rows = results[prog_name][label]
mh = sum(r["n_hard"] for r in rows) / len(rows)
ms = sum(r["n_soft"] for r in rows) / len(rows)
mt = sum(r["n_fails"] for r in rows) / len(rows)
print(f" {prog_name:14s} {label:6s}: hard={mh:.2f} soft={ms:.2f} "
f"total={mt:.2f}")
flat_hard = sum(r["n_hard"] for r in results[prog_name]["flat"]) / n_seeds
tiered_hard = sum(r["n_hard"] for r in results[prog_name]["tiered"]) / n_seeds
ok = tiered_hard <= flat_hard
overall_ok = overall_ok and ok
print(f" {prog_name:14s} hard-fail mean: flat={flat_hard:.2f} "
f"tiered={tiered_hard:.2f} -> {'PASS' if ok else 'FAIL'}")
print()
print(f"ACCEPTANCE (hard-fail mean strictly better-or-equal, both "
f"programmes): {'PASS' if overall_ok else 'FAIL'}")
print(f"wall: {time.perf_counter() - t_start:.0f}s")
print("=" * 72, flush=True)
return 0 if overall_ok else 1
if __name__ == "__main__":
sys.exit(main())

27
src/homemaker_layout/driver.py
View file

@ -131,6 +131,8 @@ class Individual:
lineage: str = "seed"
grade: float = 0.0 # §11.4 graded proximity; secondary comparator key only
sig: str = "" # §11.5 structural topology signature; niching key
n_hard: int = 0 # homemaker-py-2g7.3: hard-fail count (structural, tiered comparator)
n_soft: int = 0 # homemaker-py-2g7.3: soft-fail count (shape/quality, tiered comparator)
@dataclass
@ -187,9 +189,12 @@ def _evaluate(root: dom.Node, programme_dir, urb_root, x0, budget, inner_kw,
_fitness_for(str(programme_dir), leaf_sharing, superpose, max_share,
conn_grade, collapse_insearch, multi_use))
if pred > feasibility_max_shape_fails and pred >= best_n_fails:
# predicted_shape_fails only counts the size/width/proportion/
# crinkliness SOFT family (operators._SHAPE_FAIL_SUFFIXES), so the
# proxy carries no HARD information — tier it all soft.
ind = Individual(root=root, fitness=0.0, n_fails=pred, ratios={},
lineage=f"pruned/{lineage}", grade=0.0,
sig=genome.signature(root))
sig=genome.signature(root), n_hard=0, n_soft=pred)
return ind, 1
r = innerloop.optimise(root, programme_dir, x0=x0, budget=budget,
urb_root=urb_root, conf_overrides=overrides, **inner_kw)
@ -203,9 +208,11 @@ def _evaluate(root: dom.Node, programme_dir, urb_root, x0, budget, inner_kw,
str(programme_dir), leaf_sharing, superpose, max_share,
conn_grade, collapse_insearch, multi_use).score_with_grade(
copy.deepcopy(root))
n_hard, n_soft = fitness.tier_counts(r.fail_lines)
ind = Individual(root=root, fitness=r.fitness, n_fails=r.n_fails,
ratios=innerloop.ratio_map(root), lineage=lineage,
grade=grade, sig=genome.signature(root))
grade=grade, sig=genome.signature(root),
n_hard=n_hard, n_soft=n_soft)
return ind, r.n_evals
@ -231,6 +238,7 @@ def search(
log=None,
n_workers: int = 1,
use_lex: bool = True,
use_tiers: bool = False,
rank_bonus_fn=None,
rank_bonus_weight: float = 1.0,
seed_factory=None,
@ -395,7 +403,20 @@ def search(
# homemaker-py-qi6 §18: the connectivity signal rides the same grade channel,
# so enabling it enables the grade secondary key.
use_grade = use_grade or conn_grade
if use_lex and use_grade:
# homemaker-py-2g7.3 (DESIGN.md §37): tiered comparator, EXPERIMENT default off.
# Splits the flat -n_fails key into (-n_hard, -n_soft) so search budget stops
# being spent polishing SOFT shape fails (crinkliness/proportion/size/width/
# edge-too-long/staircase-volume) while HARD structural fails (missing space,
# wrong/required level, level/circulation/vertical connectivity, adjacency,
# stairs, covered-outside, storey limits, public access — fitness.py's
# classify_fail_tier) remain unfixed. Does not change the scalar fitness or
# total fail count, so the inner-loop 0.5^n cliff protection (§5.4) and the
# §4.9 outer A/B baseline are untouched when this flag is off.
if use_lex and use_tiers and use_grade:
_key = lambda ind: (-ind.n_hard, -ind.n_soft, ind.grade, _rank_fitness(ind))
elif use_lex and use_tiers:
_key = lambda ind: (-ind.n_hard, -ind.n_soft, _rank_fitness(ind))
elif use_lex and use_grade:
_key = lambda ind: (-ind.n_fails, ind.grade, _rank_fitness(ind))
elif use_lex:
_key = lambda ind: (-ind.n_fails, _rank_fitness(ind))

16
src/homemaker_layout/evolve.py
View file

@ -113,6 +113,20 @@ def _parse_args(argv=None) -> argparse.Namespace:
"circulation that the binary 'not connected' fail lacks. "
"Does not change the scalar fitness or fail count "
"(default: off)")
p.add_argument("--use-tiers", dest="use_tiers",
action=argparse.BooleanOptionalAction,
default=_env_bool("HOMEMAKER_USE_TIERS", False),
help="homemaker-py-2g7.3 (DESIGN.md §37): hard/soft fail "
"tiering. Outer comparator becomes (-n_hard, -n_soft, "
"fitness) instead of (-n_fails, fitness), so budget "
"stops being spent polishing SOFT shape fails "
"(crinkliness/proportion/size/width/edge-too-long/"
"staircase-volume) while HARD structural fails (missing "
"space, wrong/required level, level/circulation/vertical "
"connectivity, adjacency, stairs, covered-outside, "
"storey limits, public access) remain unfixed. Does not "
"change the scalar fitness or total fail count "
"(default: off)")
p.add_argument("--bridge-circulation", dest="bridge_circulation",
action=argparse.BooleanOptionalAction,
default=_env_bool("HOMEMAKER_BRIDGE_CIRCULATION", False),
@ -217,6 +231,7 @@ def main(argv=None) -> int:
print(f"superpose : {args.superpose}", file=sys.stderr)
print(f"multi_use : {args.multi_use}", file=sys.stderr)
print(f"conn grade : {args.conn_grade}", file=sys.stderr)
print(f"use tiers : {args.use_tiers}", file=sys.stderr)
print(f"bridge circulation : {args.bridge_circulation}", file=sys.stderr)
print(f"ruin recreate : {args.ruin_recreate}", file=sys.stderr)
print(f"collapse in-search : {args.collapse_insearch}", file=sys.stderr)
@ -271,6 +286,7 @@ def main(argv=None) -> int:
superpose=args.superpose,
multi_use=args.multi_use,
conn_grade=args.conn_grade,
use_tiers=args.use_tiers,
enable_bridge_circulation=args.bridge_circulation,
enable_ruin_recreate=args.ruin_recreate,
collapse_insearch=args.collapse_insearch,

85
src/homemaker_layout/fitness.py
View file

@ -62,6 +62,91 @@ def _leaf_grade(factors: dict[str, float]) -> float:
g += fv / FAIL_THRESHOLD
return g
# --------------------------------------------------------------------------- #
# Hard/soft fail tiering (homemaker-py-2g7.3, DESIGN.md §37)
# --------------------------------------------------------------------------- #
# HARD: the design lacks a required structural provision (a space, a level
# placement, a connectivity path, a stair, weather-tight cover) that no amount
# of ratio-only (shape) optimisation within the CURRENT topology can supply —
# fixing it needs a topology mutation (add/remove/retype/reconnect a node).
# These are graph.py's structural check_* fails plus the count/coverage fails
# fitness.py emits at the storey/building level (stairs, storey limits, public
# access, covered-outside support).
#
# SOFT: a continuous per-leaf/edge shape or quality metric — evaluate_leaf's
# perpendicular/proportion/size/width/crinkliness/access factors, wall/edge
# length caps, stair-fit volume — that the inner-loop ratio solve can, in
# principle, improve without changing the tree. "access" sits here (not with
# graph.py's structural adjacency checks) because it is computed exactly like
# proportion/crinkliness — a per-leaf continuous factor thresholded in
# evaluate_leaf — and _GRADED_FACTORS already groups it with the shape family.
#
# New fail strings MUST be added to one of these tuples — classify_fail_tier
# raises on anything unrecognised rather than silently defaulting a tier
# (homemaker-py-2g7.3 acceptance criteria).
_HARD_FAIL_MARKERS = (
"missing required space",
"too many spaces",
"would need", # missing-space cascade placeholders (size/width/proportion/
# adjacency/level/connection-below checks for an absent space)
"not adjacent to",
"on wrong level",
"not connected to", # vertical/stair connectivity to the level below
"not connected", # level circulation connectivity
"inaccessible usable space", # has_circulation disconnected a level (graph.py)
"no outside space",
"unsupported covered outside",
"covered outside above ground",
"too few stairs",
"too many stairs",
"storey limit",
"storey minimum",
"no outside public access",
)
_SOFT_FAIL_MARKERS = (
" perpendicular",
" proportion",
" size",
" width",
" crinkliness",
" access",
"edge too long",
"staircase volume",
)
def classify_fail_tier(fail: str) -> str:
"""Return ``"hard"`` or ``"soft"`` for one failure string.
Checks ``_HARD_FAIL_MARKERS`` before ``_SOFT_FAIL_MARKERS`` so cascade
placeholders like "missing k1: would need size check" (a missing-space
consequence, HARD) aren't caught by the generic " size" SOFT marker.
Raises ``ValueError`` for a fail string matching neither list.
"""
for marker in _HARD_FAIL_MARKERS:
if marker in fail:
return "hard"
for marker in _SOFT_FAIL_MARKERS:
if marker in fail:
return "soft"
raise ValueError(
f"unclassified fail string (add a tier marker in fitness.py): {fail!r}"
)
def tier_counts(fails) -> tuple[int, int]:
"""Return ``(n_hard, n_soft)`` for an iterable of failure strings."""
n_hard = n_soft = 0
for f in fails:
if classify_fail_tier(f) == "hard":
n_hard += 1
else:
n_soft += 1
return n_hard, n_soft
# Urb::Dom::Fitness::Base $CONF — keep values byte-identical to the Perl
# expressions (5.0/6 etc. evaluate to the same IEEE doubles in both languages).
CONF_DEFAULTS: dict = {

44
tests/test_driver.py
View file

@ -1,5 +1,6 @@
"""Driver tests with a faked inner loop (no oracle, no perl)."""
import copy
from pathlib import Path
import numpy as np
@ -398,3 +399,46 @@ def test_search_annealed_degenerate_ladder_falls_back(fake_inner):
assert r.best is not None
assert r.n_evals >= 300
assert all(lf.share == 1 for lf in r.best.root.leaves())
def test_use_tiers_prefers_fewer_hard_over_fewer_total_fails(monkeypatch):
"""homemaker-py-2g7.3: with use_tiers=True the outer comparator is
(-n_hard, -n_soft, fitness) instead of (-n_fails, fitness). Construct a
seed (0 hard, 2 soft) vs. a mutated child (1 hard, 0 soft, FEWER total
fails and HIGHER raw fitness) — the flat comparator prefers the child
(1 < 2 total fails); the tiered comparator must keep the seed (0 < 1
hard fails dominates regardless of soft count or fitness)."""
from homemaker_layout import innerloop
seed_root = dom.load(str(SEED_FILE))
calls = [] # first call is always the seed eval; every later call is a child
def fake_optimise(root, programme_dir, x0=None, budget=200, urb_root=None, **kw):
for _, b in innerloop.free_with_keys(root):
b.division = [0.25, 0.25]
is_seed = len(calls) == 0
calls.append(1)
if is_seed:
fail_lines = ("0/lr proportion", "0/lr crinkliness") # 0 hard, 2 soft
fit = 0.5
else:
fail_lines = ("level 0 not connected",) # 1 hard, 0 soft
fit = 0.9 # higher raw fitness AND fewer total fails than the seed
return innerloop.Result(
x=np.array([0.25]), fitness=fit, n_fails=len(fail_lines),
fail_lines=fail_lines, x0_fitness=fit, x0_n_fails=len(fail_lines),
n_evals=budget, n_oracle_calls=1,
)
monkeypatch.setattr(innerloop, "optimise", fake_optimise)
common_kw = dict(programme_dir=CORPUS, pop_size=1, seed_budget=50,
child_budget=50, budget=100, bootstrap=False, seed=0)
flat = driver.search(seed_root, **common_kw)
assert flat.best.n_fails == 1 # flat comparator: fewer total fails wins
calls.clear()
tiered = driver.search(copy.deepcopy(seed_root), use_tiers=True, **common_kw)
assert tiered.best.n_hard == 0 # tiered comparator: fewer hard fails wins
assert tiered.best.n_fails == 2

93
tests/test_fitness.py
View file

@ -10,7 +10,9 @@ from homemaker_layout.fitness import (
FAIL_THRESHOLD,
Fitness,
_leaf_grade,
classify_fail_tier,
gaussian,
tier_counts,
)
@ -375,3 +377,94 @@ def test_programme_parses_per_code_share(tmp_path):
reqs = load_programme(str(p))
assert reqs["b"].share == 3 and reqs["b"].has_share is True
assert reqs["k"].share == 1 and reqs["k"].has_share is False
# --------------------------------------------------------------------------- #
# Hard/soft fail tiering (homemaker-py-2g7.3)
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("fail_str", [
"missing required space: la1",
"missing required space: la1 (critical)",
"too many spaces: k (found 3, expected 2)",
"missing ef1: would need size check",
"missing ef1: would need width check",
"missing ef1: would need proportion check",
"missing m: would need adjacency to c",
"missing r: would need to be on level 1",
"missing t1: would need connection to c below",
"0/lr (cr1) not adjacent to c",
"li1 on wrong level (level 0, expected 1)",
"t1 not connected to c below",
"level 0 not connected",
"0 inaccessible usable space",
"level 0 no outside space",
"0/lr unsupported covered outside",
"0/lr covered outside above ground",
"too few stairs (0, min 1)",
"too many stairs (2, max 1)",
"storey limit",
"storey minimum",
"no outside public access",
])
def test_classify_fail_tier_hard(fail_str):
assert classify_fail_tier(fail_str) == "hard"
@pytest.mark.parametrize("fail_str", [
"0/lr perpendicular",
"0/lr proportion",
"0/lr size",
"0/lr width",
"0/lr crinkliness",
"0/lr access",
"0/lr lrr edge too long",
"lr outside edge too long",
"staircase volume",
])
def test_classify_fail_tier_soft(fail_str):
assert classify_fail_tier(fail_str) == "soft"
def test_classify_fail_tier_missing_cascade_is_hard_not_soft():
# "missing X: would need size check" contains the SOFT " size" substring,
# but is a consequence of a HARD missing-space fail, not a shape defect —
# the HARD markers must be checked first (fitness.py ordering).
assert classify_fail_tier("missing m#2: would need size check") == "hard"
def test_classify_fail_tier_unknown_raises():
with pytest.raises(ValueError):
classify_fail_tier("some brand new fail string nobody tiered yet")
def test_tier_counts_splits_hard_and_soft():
fails = ("level 0 not connected", "0/lr proportion", "0/lr crinkliness",
"missing required space: k1")
assert tier_counts(fails) == (2, 2)
def test_tier_counts_empty():
assert tier_counts(()) == (0, 0)
def test_classify_fail_tier_covers_full_corpus():
"""Regression guard: every fail string ever emitted into a checked-in
native (non-YAML) .fails file must still classify without error."""
import glob
from pathlib import Path
repo_root = Path(__file__).resolve().parent.parent
checked = 0
for path in glob.glob(str(repo_root / "examples" / "**" / "*.fails"), recursive=True):
with open(path) as f:
first = f.readline()
if first.startswith("---"):
continue # legacy Perl-oracle YAML .fails, not this evaluator's output
lines = [first.rstrip("\n")] + [ln.rstrip("\n") for ln in f]
for line in lines:
if not line:
continue
classify_fail_tier(line) # raises on failure
checked += 1
assert checked > 0