diff --git a/experiments/ab_9gj_ramp.py b/experiments/ab_9gj_ramp.py new file mode 100644 index 0000000..375cfab --- /dev/null +++ b/experiments/ab_9gj_ramp.py @@ -0,0 +1,231 @@ +"""Search A/B for the crinkliness tail rescale (`homemaker-py-9gj`). + +`quality_uncrinkliness` evaluates a gaussian at `x = 1/crink`, so its exponent +grows like `1/crink^2` as exposure falls. Measured over the 500k cold-start +baseline (DESIGN.md §39.12), the FAILING compact tail spans crinkliness +0.12..0.59 and quality 1e-300..1e-1 -- every value of which is numerically +zero beside a passing leaf's ~1. `crinkliness_tail="ramp"` replaces that tail, +and only that tail, with a straight line in crinkliness. + +**Why stock scoring is valid here** (the §38.9 trap, and the one case the +`9gj` bead flags as exempt): the ramp is continuous at FAIL_THRESHOLD and +strictly below it, so no leaf changes which side of the threshold it is on. +The fail set is byte-identical on every corpus artefact -- asserted in +`tests/test_fitness_crinkliness_tail.py`, not assumed here. An arm therefore +cannot win by deleting a fail category, and both arms are scored under stock. + +**Two experiment shapes**, because they answer different questions: + + --start plateau (default) seed each run from that programme's + `coldstart-500000-s.dom`. This is the ESCAPE + test the bead asks for: the ramp has signal only + where a search has already built partially-lit + rooms, and the corpus `init.dom` files show a + +0.000% score delta -- there is nothing for it to + grade at the start of a search. + --start init cold start, for comparison. + +Pairing is on (starting layout, RNG seed), so `--seeds N` over `--starts M` +gives N*M paired samples per programme. + +Usage:: + + python experiments/ab_9gj_ramp.py --budget 8000 --seeds 2 --starts 3 + python experiments/ab_9gj_ramp.py --start init --budget 8000 --seeds 6 + +Sharding, because one run is minutes and the job list is 4x that. Each shard +keeps BOTH arms of a pair together, so the two halves of a comparison never +land on differently-loaded processes:: + + for i in 0 1 2 3; do + python experiments/ab_9gj_ramp.py --budget 8000 --seeds 2 --starts 3 \ + --shard $i --nshards 4 & + done; wait + python experiments/ab_9gj_ramp.py --report + +A POWERED run needs more than the in-session pilot could afford. §39.12 puts +harbor's minimum detectable difference at n=3 at 13.7 fails; the plateau-escape +deltas here are single-digit, so budget for n >= 8 pairs per programme +(`--seeds 3 --starts 3` gives 9) and a budget large enough for either arm to +move at all -- the pilot's 8000 evals is 1.6% of what produced the plateau:: + + for i in $(seq 0 3); do + python experiments/ab_9gj_ramp.py --budget 100000 --seeds 3 --starts 3 \ + --shard $i --nshards 4 & + done; wait +""" + +from __future__ import annotations + +import argparse +import collections +import copy +import csv +import time +from pathlib import Path + +from homemaker_layout import dom as dom_mod +from homemaker_layout import driver, fitness + +CORPUS = ["examples/harbor-house", "examples/maple-court"] +ARMS = ["gaussian", "ramp"] + + +def _with_tail(tail: str): + """Patch `fitness.load_config` so every evaluator built during the run -- + the driver's, the inner loop's, the seeder's -- sees `crinkliness_tail`. + + `driver.search` has no parameter for it and `driver._fitness_for` is + lru_cached, so the cache is cleared around the patch (see ab_ssz_search). + """ + orig = fitness.load_config + + def patched(directory, overrides=None): + ov = dict(overrides or {}) + ov["crinkliness_tail"] = tail + return orig(directory, overrides=ov) + + return orig, patched + + +def tiers(fails) -> tuple[int, int]: + c = collections.Counter(fitness.classify_fail_tier(f) for f in fails) + return c["hard"], c["soft"] + + +def run_arm(progdir: str, start: Path, seed: int, tail: str, budget: int, + child_budget: int) -> dict: + orig, patched = _with_tail(tail) + fitness.load_config = patched + driver._fitness_for.cache_clear() + t0 = time.perf_counter() + try: + res = driver.search(dom_mod.load(str(start)), progdir, budget=budget, + seed=seed, child_budget=child_budget, n_workers=1) + root = copy.deepcopy(res.best.root) + finally: + fitness.load_config = orig + driver._fitness_for.cache_clear() + + conf, cost = orig(progdir, overrides={"leaf_sharing": True, + "collapse_insearch": True}) + score, fails = fitness.Fitness(conf, cost).score_with_fails(copy.deepcopy(root)) + h, s = tiers(fails) + return dict(programme=Path(progdir).name, start=start.name, seed=seed, + tail=tail, hard=h, soft=s, total=h + s, score=score, + elapsed_s=round(time.perf_counter() - t0, 1)) + + +def starting_points(progdir: str, kind: str, n: int) -> list[Path]: + """The distinct layouts to start from. + + `--starts` applies to plateau mode only: there is exactly one `init.dom`, + and repeating it would give several jobs the same (start, seed) pairing key, + which `_report` would silently collapse to one. In init mode the RNG seed + is the only sampling dimension, so use `--seeds`. + """ + d = Path(progdir) + if kind == "init": + return [d / "init.dom"] + found = sorted(d.glob("coldstart-500000-s*.dom"))[:n] + if not found: + raise SystemExit(f"no coldstart-500000-s*.dom in {d} for --start plateau") + return found + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--budget", type=int, default=8000) + ap.add_argument("--child-budget", type=int, default=80) + ap.add_argument("--seeds", type=int, default=2, help="RNG seeds per start") + ap.add_argument("--starts", type=int, default=3, + help="plateau layouts to start from (plateau mode only)") + ap.add_argument("--start", choices=("plateau", "init"), default="plateau") + ap.add_argument("--corpus", nargs="+", default=CORPUS) + ap.add_argument("--out", default="experiments/results/ab_9gj_ramp.csv") + ap.add_argument("--shard", type=int, default=0, + help="run only jobs i where i %% nshards == shard") + ap.add_argument("--nshards", type=int, default=1, + help="split the job list across N processes; each writes " + ".shard. Use --report to merge and analyse.") + ap.add_argument("--report", action="store_true", + help="merge .shard* (or ) and print the analysis " + "only -- runs nothing") + args = ap.parse_args() + + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + + if args.report: + rows = [] + shards = sorted(out.parent.glob(out.name + ".shard*")) or ( + [out] if out.exists() else []) + for sh in shards: + with sh.open() as fh: + for r in csv.DictReader(fh): + for k in ("total", "hard", "soft", "seed"): + r[k] = int(r[k]) + r["score"] = float(r["score"]) + rows.append(r) + if len(shards) > 1: + with out.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + _report(rows, args.corpus) + print(f"\nmerged {len(rows)} runs from {len(shards)} file(s) into {out}") + return + + # Build the whole job list first so sharding is deterministic and every + # (start, seed) pair keeps BOTH arms in the same shard -- the comparison is + # paired, and splitting a pair across processes would let machine load + # differ between the two halves of one pair. + jobs = [] + for progdir in args.corpus: + for start in starting_points(progdir, args.start, args.starts): + for seed in range(args.seeds): + jobs.append((progdir, start, seed)) + + rows: list[dict] = [] + dest = (out if args.nshards == 1 + else out.with_name(out.name + f".shard{args.shard}")) + for i, (progdir, start, seed) in enumerate(jobs): + if i % args.nshards != args.shard: + continue + for tail in ARMS: + r = run_arm(progdir, start, seed, tail, args.budget, + args.child_budget) + rows.append(r) + print(f" {r['programme']:<14} {start.name:<26} seed={seed} " + f"{tail:<9} {r['hard']}h/{r['soft']}s = {r['total']:3d} " + f"score {r['score']:.4g} {r['elapsed_s']}s", flush=True) + with dest.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + if args.nshards == 1: + _report(rows, args.corpus) + print(f"\nwrote {dest}") + + +def _report(rows, corpus) -> None: + """homemaker-py-tco: state what this N could resolve, beside the result.""" + from ab_report import format_report, paired_report + for progdir in corpus: + name = Path(progdir).name + by: dict[tuple, dict] = {} + for r in rows: + if r["programme"] == name: + by.setdefault((r["start"], r["seed"]), {})[r["tail"]] = r["total"] + keys = sorted(k for k, v in by.items() if len(v) == 2) + if len(keys) < 2: + continue + print(f"\n--- {name}: ramp vs gaussian (stock-scored total fails) ---") + print(format_report(paired_report( + [by[k]["gaussian"] for k in keys], [by[k]["ramp"] for k in keys], + "gaussian", "ramp"))) + + +if __name__ == "__main__": + main() diff --git a/src/homemaker_layout/fitness.py b/src/homemaker_layout/fitness.py index 2d02877..65a6a83 100644 --- a/src/homemaker_layout/fitness.py +++ b/src/homemaker_layout/fitness.py @@ -20,6 +20,8 @@ Call ``dom.merge_divided(root)`` and rebuild graphs before ``process_storey`` from __future__ import annotations +import functools +import math from dataclasses import dataclass, field from pathlib import Path @@ -144,7 +146,6 @@ def connectivity_weight_for(value_inside: float, value_circulation: float) -> fl Derived from the value rates rather than hard-coded, so the two stay in step if either rate is ever retuned. """ - import math if value_inside <= 0 or value_circulation <= 0: return 1.0 ratio = value_circulation / value_inside @@ -243,6 +244,22 @@ def gaussian(x: float, a: float, b: float, c: float) -> float: return a * (_E ** (0 - ((x - b) ** 2 / (2 * c * c)))) +@functools.lru_cache(maxsize=None) +def _crink_at_fail_threshold(distance: float, sigma: float) -> float: + """The crinkliness at which the stock gaussian crosses ``FAIL_THRESHOLD`` + on the COMPACT (too-little-exposure) side (homemaker-py-9gj). + + ``quality_uncrinkliness`` evaluates the gaussian at ``x = 1/crink``, so + solving ``gaussian(x, 1, distance, sigma) == FAIL_THRESHOLD`` for the root + above ``distance`` and inverting gives the crinkliness below which a leaf + fails. Uses ``_E`` rather than ``math.e`` so the crossing agrees with the + truncated constant the gaussian itself is evaluated with. + """ + d = math.sqrt(2.0 * sigma * sigma + * math.log(1.0 / FAIL_THRESHOLD) / math.log(_E)) + return 1.0 / (distance + d) + + def _gaussian_product(target_a: float, sigma_a: float, target_b: float, sigma_b: float) -> tuple[float, float]: """Precision-weighted combination of two Gaussian (target, sigma) pairs @@ -448,6 +465,25 @@ class Fitness: # would silently delete a whole fail category. self._crinkliness_floor = float( self.conf("crinkliness_floor") or 0.01) + # homemaker-py-9gj (DESIGN.md §39.13): how the FAILING compact tail of + # the crinkliness gaussian is scaled. "gaussian" (default) is stock, + # byte-identical to every prior run. "ramp" replaces the tail — and + # only the tail, only below FAIL_THRESHOLD, only on the compact side — + # with a straight line in crinkliness, because the stock exponent grows + # like 1/crink^2 and underflows the whole tail to a numerically + # indistinguishable zero. Nothing at or above FAIL_THRESHOLD moves, so + # the fail set is byte-identical either way and no calibration changes. + self._crinkliness_tail = str(self.conf("crinkliness_tail") or "gaussian") + if self._crinkliness_tail not in ("gaussian", "ramp"): + raise ValueError( + f"unknown crinkliness_tail: {self._crinkliness_tail!r}") + if self._crinkliness_tail == "ramp" and self._crinkliness_mode != "urb": + # Both rewrite the same tail; composing them would give a shape + # neither was measured under. + raise ValueError( + "crinkliness_tail='ramp' is incompatible with " + f"crinkliness_mode={self._crinkliness_mode!r} (§38.1's modes are " + "superseded; use one or the other, not both)") def usages(self) -> dict[str, str]: """``{room code: usage}`` for this programme (homemaker-py-sel). @@ -1257,7 +1293,22 @@ class Fitness: q = gaussian(1 / crink, 1.0, distance, sigma) if one_sided and 1 / crink > distance: return 1.0 - return max(q, self._crinkliness_floor) if mode in ("floor", "compact_ok") else q + if mode in ("floor", "compact_ok"): + return max(q, self._crinkliness_floor) + if (self._crinkliness_tail == "ramp" + and q < FAIL_THRESHOLD and 1 / crink > distance): + # homemaker-py-9gj (DESIGN.md §39.13). The stock exponent is + # (1/crink - distance)^2 / 2sigma^2, so it grows without bound as + # exposure falls: measured over the corpus baseline, the failing + # compact tail spans crink 0.12..0.59 and q 1e-300..1e-1, every + # value of which is numerically zero beside a passing leaf's ~1. + # The search therefore cannot rank two layouts that differ only in + # how exposed their under-lit rooms are. Replacing the tail with a + # straight line in crink keeps the ordering, makes it + # representable, is continuous at the threshold, and still sends a + # fully buried leaf (crink == 0, handled above) to exactly 0. + return FAIL_THRESHOLD * crink / _crink_at_fail_threshold(distance, sigma) + return q # --- access --- # diff --git a/tests/test_fitness_crinkliness_tail.py b/tests/test_fitness_crinkliness_tail.py new file mode 100644 index 0000000..d8b0dbe --- /dev/null +++ b/tests/test_fitness_crinkliness_tail.py @@ -0,0 +1,113 @@ +"""The `crinkliness_tail="ramp"` rescale (homemaker-py-9gj, DESIGN.md §39.13). + +The whole design rests on one invariant: the ramp rewrites the failing compact +tail and NOTHING else, so no leaf crosses FAIL_THRESHOLD and the fail set is +byte-identical to stock. That is what makes it legal to score both arms of the +A/B under the stock objective (the §38.9 trap's one exemption). It is asserted +here on every committed corpus artefact rather than assumed. +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest + +from homemaker_layout import dom as dom_mod +from homemaker_layout.fitness import ( + FAIL_THRESHOLD, Fitness, _crink_at_fail_threshold, gaussian, load_config, +) + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" +PROGRAMMES = ["harbor-house", "maple-court", "health-centre", "programme-house"] + + +def _artefacts(): + for name in PROGRAMMES: + d = EXAMPLES / name + if not d.is_dir(): + continue + for p in sorted(d.glob("coldstart-500000-s*.dom")) + \ + sorted(d.glob("evolved-3M*.dom")) + [d / "init.dom"]: + if p.exists(): + yield d, p + + +def test_crossing_is_continuous_at_the_fail_threshold(): + """The ramp meets the gaussian exactly at FAIL_THRESHOLD, so the factor is + continuous there and the ordering across the boundary is preserved.""" + for distance, sigma in ((5.0 / 6, 1.1 / 3), (1.2, 0.25), (0.5, 0.5)): + c0 = _crink_at_fail_threshold(distance, sigma) + assert gaussian(1 / c0, 1.0, distance, sigma) == pytest.approx( + FAIL_THRESHOLD, rel=1e-12) + # and it is the COMPACT-side root: less exposure than c0, not more + assert 1 / c0 > distance + + +def test_ramp_is_strictly_monotone_where_the_gaussian_has_underflowed(): + """The point of the change. Stock assigns the same double -- 0.0 -- to + every leaf below crink ~= 1/15; the ramp separates them.""" + distance, sigma = 5.0 / 6, 1.1 / 3 + c0 = _crink_at_fail_threshold(distance, sigma) + crinks = [0.0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, c0 * 0.999] + + stock = [gaussian(1 / c, 1.0, distance, sigma) if c else 0.0 for c in crinks] + ramp = [FAIL_THRESHOLD * c / c0 for c in crinks] + + assert len(set(stock)) < len(set(ramp)), "stock should collapse values the ramp keeps" + assert stock.count(0.0) > 1, "the flat-zero region is what this fixes" + assert all(b > a for a, b in zip(ramp, ramp[1:])), "ramp must be strictly increasing" + assert ramp[0] == 0.0, "a fully buried leaf is still worth nothing" + assert all(q < FAIL_THRESHOLD for q in ramp), "the ramp must never lift a leaf out of failing" + + +@pytest.mark.skipif(not (EXAMPLES / "harbor-house").is_dir(), + reason="examples absent") +def test_fail_set_is_byte_identical_across_the_corpus(): + seen = 0 + for d, p in _artefacts(): + root = dom_mod.load(str(p)) + c_stock, cost = load_config(d) + c_ramp, _ = load_config(d, overrides={"crinkliness_tail": "ramp"}) + _, f_stock = Fitness(c_stock, cost).score_with_fails(copy.deepcopy(root)) + _, f_ramp = Fitness(c_ramp, cost).score_with_fails(copy.deepcopy(root)) + assert f_stock == f_ramp, f"{p} changed its fail set under the ramp" + seen += 1 + assert seen >= 4, "expected to have checked several corpus artefacts" + + +@pytest.mark.skipif(not (EXAMPLES / "harbor-house").is_dir(), + reason="examples absent") +def test_ramp_never_lowers_the_score(): + """Every affected factor rises (0 or ~0 -> a representable fraction of + FAIL_THRESHOLD), and quality is a product with value accumulating + positively, so the scalar can only go up or stay put.""" + for d, p in _artefacts(): + root = dom_mod.load(str(p)) + c_stock, cost = load_config(d) + c_ramp, _ = load_config(d, overrides={"crinkliness_tail": "ramp"}) + s_stock, _ = Fitness(c_stock, cost).score_with_fails(copy.deepcopy(root)) + s_ramp, _ = Fitness(c_ramp, cost).score_with_fails(copy.deepcopy(root)) + assert s_ramp >= s_stock, f"{p} scored lower under the ramp" + + +def test_ramp_refuses_to_compose_with_the_superseded_modes(): + """§38.1's modes rewrite the same tail; stacking them would give a shape + neither was measured under.""" + d = EXAMPLES / "harbor-house" + if not d.is_dir(): + pytest.skip("examples absent") + conf, cost = load_config(d, overrides={"crinkliness_tail": "ramp", + "crinkliness_mode": "floor"}) + with pytest.raises(ValueError, match="incompatible"): + Fitness(conf, cost) + + +def test_unknown_tail_is_rejected(): + d = EXAMPLES / "harbor-house" + if not d.is_dir(): + pytest.skip("examples absent") + conf, cost = load_config(d, overrides={"crinkliness_tail": "linear"}) + with pytest.raises(ValueError, match="unknown crinkliness_tail"): + Fitness(conf, cost)