From cd392e77c586cb7400b322f85505373be5c9a53d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 06:29:02 +0000 Subject: [PATCH] Rescale the underflowing crinkliness tail so the failing region has an ordering quality_uncrinkliness evaluates a gaussian at x = 1/crink, so its exponent grows like 1/crink^2 and underflows a double to exactly zero below crink ~ 1/15. Measured over the twelve 500k cold-start runs (39.12): 430 leaves carry a minimum-exposure requirement, 112 fail it, and those 112 span quality 1e-300..1e-1 while contributing 0.034% of total value on 23% of the floor area. Every value in that range is numerically zero beside a passing leaf's ~1, so the search cannot rank two layouts that differ only in how exposed their under-lit rooms are. This is wider than the bead's diagnosis (a flat 0.0 for zero-exposure leaves) and it explains why 38.1's `floor` mode measured as a no-op: max(q, 0.01) maps 110 of the 112 onto one constant, replacing a flat zero with a flat 0.01. crinkliness_tail="ramp" (default OFF, "gaussian" is stock) replaces the tail -- only the tail, only below FAIL_THRESHOLD, only on the compact side -- with a straight line in crinkliness meeting the gaussian exactly at the crossing. _crink_at_fail_threshold inverts the gaussian there using the same truncated _E the factor is evaluated with. Deliberately conservative: nothing at or above FAIL_THRESHOLD moves, so no calibration changes and no leaf crosses the threshold. The fail set is byte-identical on all 21 committed corpus artefacts, the four init.dom seeds included -- asserted in tests/test_fitness_crinkliness_tail.py, not assumed. That invariance is also what makes it legal to score both arms of the A/B under stock (the 38.9 trap's one exemption). A fully buried leaf still scores exactly 0; this restores an ordering within the failing region, it does not forgive it. Composing with 38.1's superseded modes is refused, since both rewrite the same tail. Score effect on the baseline artefacts: +0.3%..+2.8% on harbor and maple, exactly +0.000% on health-centre, programme-house, and every init.dom -- a programme with no partially-exposed failing rooms has nothing to grade, and neither does any starting layout. The ramp is a mid-search signal by construction, so experiments/ab_9gj_ramp.py defaults to seeding each run from a 500k plateau artefact rather than cold. The module-level math import replaces a now-redundant local one. DESIGN.md 39.13 and the A/B verdict follow in a separate commit. Refs homemaker-py-9gj. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MJ84Feep79Hhm3E4zZJmnB --- experiments/ab_9gj_ramp.py | 231 +++++++++++++++++++++++++ src/homemaker_layout/fitness.py | 55 +++++- tests/test_fitness_crinkliness_tail.py | 113 ++++++++++++ 3 files changed, 397 insertions(+), 2 deletions(-) create mode 100644 experiments/ab_9gj_ramp.py create mode 100644 tests/test_fitness_crinkliness_tail.py diff --git a/experiments/ab_9gj_ramp.py b/experiments/ab_9gj_ramp.py new file mode 100644 index 0000000..375cfab --- /dev/null +++ b/experiments/ab_9gj_ramp.py @@ -0,0 +1,231 @@ +"""Search A/B for the crinkliness tail rescale (`homemaker-py-9gj`). + +`quality_uncrinkliness` evaluates a gaussian at `x = 1/crink`, so its exponent +grows like `1/crink^2` as exposure falls. Measured over the 500k cold-start +baseline (DESIGN.md §39.12), the FAILING compact tail spans crinkliness +0.12..0.59 and quality 1e-300..1e-1 -- every value of which is numerically +zero beside a passing leaf's ~1. `crinkliness_tail="ramp"` replaces that tail, +and only that tail, with a straight line in crinkliness. + +**Why stock scoring is valid here** (the §38.9 trap, and the one case the +`9gj` bead flags as exempt): the ramp is continuous at FAIL_THRESHOLD and +strictly below it, so no leaf changes which side of the threshold it is on. +The fail set is byte-identical on every corpus artefact -- asserted in +`tests/test_fitness_crinkliness_tail.py`, not assumed here. An arm therefore +cannot win by deleting a fail category, and both arms are scored under stock. + +**Two experiment shapes**, because they answer different questions: + + --start plateau (default) seed each run from that programme's + `coldstart-500000-s.dom`. This is the ESCAPE + test the bead asks for: the ramp has signal only + where a search has already built partially-lit + rooms, and the corpus `init.dom` files show a + +0.000% score delta -- there is nothing for it to + grade at the start of a search. + --start init cold start, for comparison. + +Pairing is on (starting layout, RNG seed), so `--seeds N` over `--starts M` +gives N*M paired samples per programme. + +Usage:: + + python experiments/ab_9gj_ramp.py --budget 8000 --seeds 2 --starts 3 + python experiments/ab_9gj_ramp.py --start init --budget 8000 --seeds 6 + +Sharding, because one run is minutes and the job list is 4x that. Each shard +keeps BOTH arms of a pair together, so the two halves of a comparison never +land on differently-loaded processes:: + + for i in 0 1 2 3; do + python experiments/ab_9gj_ramp.py --budget 8000 --seeds 2 --starts 3 \ + --shard $i --nshards 4 & + done; wait + python experiments/ab_9gj_ramp.py --report + +A POWERED run needs more than the in-session pilot could afford. §39.12 puts +harbor's minimum detectable difference at n=3 at 13.7 fails; the plateau-escape +deltas here are single-digit, so budget for n >= 8 pairs per programme +(`--seeds 3 --starts 3` gives 9) and a budget large enough for either arm to +move at all -- the pilot's 8000 evals is 1.6% of what produced the plateau:: + + for i in $(seq 0 3); do + python experiments/ab_9gj_ramp.py --budget 100000 --seeds 3 --starts 3 \ + --shard $i --nshards 4 & + done; wait +""" + +from __future__ import annotations + +import argparse +import collections +import copy +import csv +import time +from pathlib import Path + +from homemaker_layout import dom as dom_mod +from homemaker_layout import driver, fitness + +CORPUS = ["examples/harbor-house", "examples/maple-court"] +ARMS = ["gaussian", "ramp"] + + +def _with_tail(tail: str): + """Patch `fitness.load_config` so every evaluator built during the run -- + the driver's, the inner loop's, the seeder's -- sees `crinkliness_tail`. + + `driver.search` has no parameter for it and `driver._fitness_for` is + lru_cached, so the cache is cleared around the patch (see ab_ssz_search). + """ + orig = fitness.load_config + + def patched(directory, overrides=None): + ov = dict(overrides or {}) + ov["crinkliness_tail"] = tail + return orig(directory, overrides=ov) + + return orig, patched + + +def tiers(fails) -> tuple[int, int]: + c = collections.Counter(fitness.classify_fail_tier(f) for f in fails) + return c["hard"], c["soft"] + + +def run_arm(progdir: str, start: Path, seed: int, tail: str, budget: int, + child_budget: int) -> dict: + orig, patched = _with_tail(tail) + fitness.load_config = patched + driver._fitness_for.cache_clear() + t0 = time.perf_counter() + try: + res = driver.search(dom_mod.load(str(start)), progdir, budget=budget, + seed=seed, child_budget=child_budget, n_workers=1) + root = copy.deepcopy(res.best.root) + finally: + fitness.load_config = orig + driver._fitness_for.cache_clear() + + conf, cost = orig(progdir, overrides={"leaf_sharing": True, + "collapse_insearch": True}) + score, fails = fitness.Fitness(conf, cost).score_with_fails(copy.deepcopy(root)) + h, s = tiers(fails) + return dict(programme=Path(progdir).name, start=start.name, seed=seed, + tail=tail, hard=h, soft=s, total=h + s, score=score, + elapsed_s=round(time.perf_counter() - t0, 1)) + + +def starting_points(progdir: str, kind: str, n: int) -> list[Path]: + """The distinct layouts to start from. + + `--starts` applies to plateau mode only: there is exactly one `init.dom`, + and repeating it would give several jobs the same (start, seed) pairing key, + which `_report` would silently collapse to one. In init mode the RNG seed + is the only sampling dimension, so use `--seeds`. + """ + d = Path(progdir) + if kind == "init": + return [d / "init.dom"] + found = sorted(d.glob("coldstart-500000-s*.dom"))[:n] + if not found: + raise SystemExit(f"no coldstart-500000-s*.dom in {d} for --start plateau") + return found + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--budget", type=int, default=8000) + ap.add_argument("--child-budget", type=int, default=80) + ap.add_argument("--seeds", type=int, default=2, help="RNG seeds per start") + ap.add_argument("--starts", type=int, default=3, + help="plateau layouts to start from (plateau mode only)") + ap.add_argument("--start", choices=("plateau", "init"), default="plateau") + ap.add_argument("--corpus", nargs="+", default=CORPUS) + ap.add_argument("--out", default="experiments/results/ab_9gj_ramp.csv") + ap.add_argument("--shard", type=int, default=0, + help="run only jobs i where i %% nshards == shard") + ap.add_argument("--nshards", type=int, default=1, + help="split the job list across N processes; each writes " + ".shard. Use --report to merge and analyse.") + ap.add_argument("--report", action="store_true", + help="merge .shard* (or ) and print the analysis " + "only -- runs nothing") + args = ap.parse_args() + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + + if args.report: + rows = [] + shards = sorted(out.parent.glob(out.name + ".shard*")) or ( + [out] if out.exists() else []) + for sh in shards: + with sh.open() as fh: + for r in csv.DictReader(fh): + for k in ("total", "hard", "soft", "seed"): + r[k] = int(r[k]) + r["score"] = float(r["score"]) + rows.append(r) + if len(shards) > 1: + with out.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + _report(rows, args.corpus) + print(f"\nmerged {len(rows)} runs from {len(shards)} file(s) into {out}") + return + + # Build the whole job list first so sharding is deterministic and every + # (start, seed) pair keeps BOTH arms in the same shard -- the comparison is + # paired, and splitting a pair across processes would let machine load + # differ between the two halves of one pair. + jobs = [] + for progdir in args.corpus: + for start in starting_points(progdir, args.start, args.starts): + for seed in range(args.seeds): + jobs.append((progdir, start, seed)) + + rows: list[dict] = [] + dest = (out if args.nshards == 1 + else out.with_name(out.name + f".shard{args.shard}")) + for i, (progdir, start, seed) in enumerate(jobs): + if i % args.nshards != args.shard: + continue + for tail in ARMS: + r = run_arm(progdir, start, seed, tail, args.budget, + args.child_budget) + rows.append(r) + print(f" {r['programme']:<14} {start.name:<26} seed={seed} " + f"{tail:<9} {r['hard']}h/{r['soft']}s = {r['total']:3d} " + f"score {r['score']:.4g} {r['elapsed_s']}s", flush=True) + with dest.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + if args.nshards == 1: + _report(rows, args.corpus) + print(f"\nwrote {dest}") + + +def _report(rows, corpus) -> None: + """homemaker-py-tco: state what this N could resolve, beside the result.""" + from ab_report import format_report, paired_report + for progdir in corpus: + name = Path(progdir).name + by: dict[tuple, dict] = {} + for r in rows: + if r["programme"] == name: + by.setdefault((r["start"], r["seed"]), {})[r["tail"]] = r["total"] + keys = sorted(k for k, v in by.items() if len(v) == 2) + if len(keys) < 2: + continue + print(f"\n--- {name}: ramp vs gaussian (stock-scored total fails) ---") + print(format_report(paired_report( + [by[k]["gaussian"] for k in keys], [by[k]["ramp"] for k in keys], + "gaussian", "ramp"))) + + +if __name__ == "__main__": + main() diff --git a/src/homemaker_layout/fitness.py b/src/homemaker_layout/fitness.py index 2d02877..65a6a83 100644 --- a/src/homemaker_layout/fitness.py +++ b/src/homemaker_layout/fitness.py @@ -20,6 +20,8 @@ Call ``dom.merge_divided(root)`` and rebuild graphs before ``process_storey`` from __future__ import annotations +import functools +import math from dataclasses import dataclass, field from pathlib import Path @@ -144,7 +146,6 @@ def connectivity_weight_for(value_inside: float, value_circulation: float) -> fl Derived from the value rates rather than hard-coded, so the two stay in step if either rate is ever retuned. """ - import math if value_inside <= 0 or value_circulation <= 0: return 1.0 ratio = value_circulation / value_inside @@ -243,6 +244,22 @@ def gaussian(x: float, a: float, b: float, c: float) -> float: return a * (_E ** (0 - ((x - b) ** 2 / (2 * c * c)))) +@functools.lru_cache(maxsize=None) +def _crink_at_fail_threshold(distance: float, sigma: float) -> float: + """The crinkliness at which the stock gaussian crosses ``FAIL_THRESHOLD`` + on the COMPACT (too-little-exposure) side (homemaker-py-9gj). + + ``quality_uncrinkliness`` evaluates the gaussian at ``x = 1/crink``, so + solving ``gaussian(x, 1, distance, sigma) == FAIL_THRESHOLD`` for the root + above ``distance`` and inverting gives the crinkliness below which a leaf + fails. Uses ``_E`` rather than ``math.e`` so the crossing agrees with the + truncated constant the gaussian itself is evaluated with. + """ + d = math.sqrt(2.0 * sigma * sigma + * math.log(1.0 / FAIL_THRESHOLD) / math.log(_E)) + return 1.0 / (distance + d) + + def _gaussian_product(target_a: float, sigma_a: float, target_b: float, sigma_b: float) -> tuple[float, float]: """Precision-weighted combination of two Gaussian (target, sigma) pairs @@ -448,6 +465,25 @@ class Fitness: # would silently delete a whole fail category. self._crinkliness_floor = float( self.conf("crinkliness_floor") or 0.01) + # homemaker-py-9gj (DESIGN.md §39.13): how the FAILING compact tail of + # the crinkliness gaussian is scaled. "gaussian" (default) is stock, + # byte-identical to every prior run. "ramp" replaces the tail — and + # only the tail, only below FAIL_THRESHOLD, only on the compact side — + # with a straight line in crinkliness, because the stock exponent grows + # like 1/crink^2 and underflows the whole tail to a numerically + # indistinguishable zero. Nothing at or above FAIL_THRESHOLD moves, so + # the fail set is byte-identical either way and no calibration changes. + self._crinkliness_tail = str(self.conf("crinkliness_tail") or "gaussian") + if self._crinkliness_tail not in ("gaussian", "ramp"): + raise ValueError( + f"unknown crinkliness_tail: {self._crinkliness_tail!r}") + if self._crinkliness_tail == "ramp" and self._crinkliness_mode != "urb": + # Both rewrite the same tail; composing them would give a shape + # neither was measured under. + raise ValueError( + "crinkliness_tail='ramp' is incompatible with " + f"crinkliness_mode={self._crinkliness_mode!r} (§38.1's modes are " + "superseded; use one or the other, not both)") def usages(self) -> dict[str, str]: """``{room code: usage}`` for this programme (homemaker-py-sel). @@ -1257,7 +1293,22 @@ class Fitness: q = gaussian(1 / crink, 1.0, distance, sigma) if one_sided and 1 / crink > distance: return 1.0 - return max(q, self._crinkliness_floor) if mode in ("floor", "compact_ok") else q + if mode in ("floor", "compact_ok"): + return max(q, self._crinkliness_floor) + if (self._crinkliness_tail == "ramp" + and q < FAIL_THRESHOLD and 1 / crink > distance): + # homemaker-py-9gj (DESIGN.md §39.13). The stock exponent is + # (1/crink - distance)^2 / 2sigma^2, so it grows without bound as + # exposure falls: measured over the corpus baseline, the failing + # compact tail spans crink 0.12..0.59 and q 1e-300..1e-1, every + # value of which is numerically zero beside a passing leaf's ~1. + # The search therefore cannot rank two layouts that differ only in + # how exposed their under-lit rooms are. Replacing the tail with a + # straight line in crink keeps the ordering, makes it + # representable, is continuous at the threshold, and still sends a + # fully buried leaf (crink == 0, handled above) to exactly 0. + return FAIL_THRESHOLD * crink / _crink_at_fail_threshold(distance, sigma) + return q # --- access --- # diff --git a/tests/test_fitness_crinkliness_tail.py b/tests/test_fitness_crinkliness_tail.py new file mode 100644 index 0000000..d8b0dbe --- /dev/null +++ b/tests/test_fitness_crinkliness_tail.py @@ -0,0 +1,113 @@ +"""The `crinkliness_tail="ramp"` rescale (homemaker-py-9gj, DESIGN.md §39.13). + +The whole design rests on one invariant: the ramp rewrites the failing compact +tail and NOTHING else, so no leaf crosses FAIL_THRESHOLD and the fail set is +byte-identical to stock. That is what makes it legal to score both arms of the +A/B under the stock objective (the §38.9 trap's one exemption). It is asserted +here on every committed corpus artefact rather than assumed. +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest + +from homemaker_layout import dom as dom_mod +from homemaker_layout.fitness import ( + FAIL_THRESHOLD, Fitness, _crink_at_fail_threshold, gaussian, load_config, +) + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" +PROGRAMMES = ["harbor-house", "maple-court", "health-centre", "programme-house"] + + +def _artefacts(): + for name in PROGRAMMES: + d = EXAMPLES / name + if not d.is_dir(): + continue + for p in sorted(d.glob("coldstart-500000-s*.dom")) + \ + sorted(d.glob("evolved-3M*.dom")) + [d / "init.dom"]: + if p.exists(): + yield d, p + + +def test_crossing_is_continuous_at_the_fail_threshold(): + """The ramp meets the gaussian exactly at FAIL_THRESHOLD, so the factor is + continuous there and the ordering across the boundary is preserved.""" + for distance, sigma in ((5.0 / 6, 1.1 / 3), (1.2, 0.25), (0.5, 0.5)): + c0 = _crink_at_fail_threshold(distance, sigma) + assert gaussian(1 / c0, 1.0, distance, sigma) == pytest.approx( + FAIL_THRESHOLD, rel=1e-12) + # and it is the COMPACT-side root: less exposure than c0, not more + assert 1 / c0 > distance + + +def test_ramp_is_strictly_monotone_where_the_gaussian_has_underflowed(): + """The point of the change. Stock assigns the same double -- 0.0 -- to + every leaf below crink ~= 1/15; the ramp separates them.""" + distance, sigma = 5.0 / 6, 1.1 / 3 + c0 = _crink_at_fail_threshold(distance, sigma) + crinks = [0.0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, c0 * 0.999] + + stock = [gaussian(1 / c, 1.0, distance, sigma) if c else 0.0 for c in crinks] + ramp = [FAIL_THRESHOLD * c / c0 for c in crinks] + + assert len(set(stock)) < len(set(ramp)), "stock should collapse values the ramp keeps" + assert stock.count(0.0) > 1, "the flat-zero region is what this fixes" + assert all(b > a for a, b in zip(ramp, ramp[1:])), "ramp must be strictly increasing" + assert ramp[0] == 0.0, "a fully buried leaf is still worth nothing" + assert all(q < FAIL_THRESHOLD for q in ramp), "the ramp must never lift a leaf out of failing" + + +@pytest.mark.skipif(not (EXAMPLES / "harbor-house").is_dir(), + reason="examples absent") +def test_fail_set_is_byte_identical_across_the_corpus(): + seen = 0 + for d, p in _artefacts(): + root = dom_mod.load(str(p)) + c_stock, cost = load_config(d) + c_ramp, _ = load_config(d, overrides={"crinkliness_tail": "ramp"}) + _, f_stock = Fitness(c_stock, cost).score_with_fails(copy.deepcopy(root)) + _, f_ramp = Fitness(c_ramp, cost).score_with_fails(copy.deepcopy(root)) + assert f_stock == f_ramp, f"{p} changed its fail set under the ramp" + seen += 1 + assert seen >= 4, "expected to have checked several corpus artefacts" + + +@pytest.mark.skipif(not (EXAMPLES / "harbor-house").is_dir(), + reason="examples absent") +def test_ramp_never_lowers_the_score(): + """Every affected factor rises (0 or ~0 -> a representable fraction of + FAIL_THRESHOLD), and quality is a product with value accumulating + positively, so the scalar can only go up or stay put.""" + for d, p in _artefacts(): + root = dom_mod.load(str(p)) + c_stock, cost = load_config(d) + c_ramp, _ = load_config(d, overrides={"crinkliness_tail": "ramp"}) + s_stock, _ = Fitness(c_stock, cost).score_with_fails(copy.deepcopy(root)) + s_ramp, _ = Fitness(c_ramp, cost).score_with_fails(copy.deepcopy(root)) + assert s_ramp >= s_stock, f"{p} scored lower under the ramp" + + +def test_ramp_refuses_to_compose_with_the_superseded_modes(): + """§38.1's modes rewrite the same tail; stacking them would give a shape + neither was measured under.""" + d = EXAMPLES / "harbor-house" + if not d.is_dir(): + pytest.skip("examples absent") + conf, cost = load_config(d, overrides={"crinkliness_tail": "ramp", + "crinkliness_mode": "floor"}) + with pytest.raises(ValueError, match="incompatible"): + Fitness(conf, cost) + + +def test_unknown_tail_is_rejected(): + d = EXAMPLES / "harbor-house" + if not d.is_dir(): + pytest.skip("examples absent") + conf, cost = load_config(d, overrides={"crinkliness_tail": "linear"}) + with pytest.raises(ValueError, match="unknown crinkliness_tail"): + Fitness(conf, cost)