Bruno Postle
7e39bf5870
24-run sweep (maple-court + harbor, seeds 0/1/2, 20000 evals): M3 reassociate and the shape-feasibility filter are both neutral-to-slightly-worse vs the §12.2 baseline (maple 136.0 -> 139-140, harbor 74.0 -> 77-78). Baseline controls reproduce §12.2 exactly, so the negative is real. Verdict: the Phase-7 residual is the geometry/shape floor of the constructed slicing layouts, not reachability/feasibility-bound — third independent negative on search machinery (§11.4/§11.5/§12.3) vs four construction/seed wins (§11.2/§11.6/§11.7/§12.2). A full canonical Polish rewrite is not justified: its one testable promise (associativity reachability) was tested and did not pay. Both operators kept default-OFF. Closes 9gp.1, 9gp.2, 9gp; epic leu (Phase 7) auto-closed (3/3). Adds the reproducible sweep harness experiments/run_9gp_ab.sh. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
60 lines
2.4 KiB
Bash
Executable file
60 lines
2.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# 9gp A/B sweep (DESIGN.md §12.3): baseline vs reassociate vs shape-feasibility
|
|
# vs combined, on the maple-court scaling benchmark + harbor, seeds 0/1/2.
|
|
#
|
|
# Authoritative metric = the re-scored native fail count (the "re-scored
|
|
# (native): <fit> (<n> fails) -> OK" line). Each run appends one TSV row to
|
|
# scratch/9gp_ab_results.tsv so partial results survive an interrupt.
|
|
#
|
|
# Usage: experiments/run_9gp_ab.sh [budget] [seeds...]
|
|
set -u
|
|
cd "$(dirname "$0")/.."
|
|
|
|
BUDGET="${1:-20000}"; shift || true
|
|
SEEDS=("${@:-0 1 2}")
|
|
# shellcheck disable=SC2206
|
|
SEEDS=(${SEEDS[*]})
|
|
|
|
OUT=scratch/9gp_ab
|
|
mkdir -p "$OUT"
|
|
TSV=scratch/9gp_ab_results.tsv
|
|
[ -f "$TSV" ] || printf 'programme\tseed\tconfig\tfails\tevals\ttopologies\telapsed_s\n' > "$TSV"
|
|
|
|
# per-programme prune threshold (below the incumbent range; the pred>=best guard
|
|
# in driver._evaluate dominates — see DESIGN.md §12.3 calibration).
|
|
declare -A MAXSHAPE=( [maple-court]=100 [harbor-house]=55 )
|
|
|
|
run() { # programme seed config_name ENVVARS...
|
|
local prog="$1" seed="$2" cfg="$3"; shift 3
|
|
local dom="examples/$prog/init.dom"
|
|
local out="$OUT/${prog}_${cfg}_s${seed}.dom"
|
|
local log="$OUT/${prog}_${cfg}_s${seed}.log"
|
|
echo ">>> $prog seed=$seed config=$cfg ($*)"
|
|
local t0; t0=$(date +%s)
|
|
env URB_NO_OCCLUSION=1 "$@" \
|
|
python3 experiments/run_staged_search.py "examples/$prog" "$BUDGET" "$seed" "$dom" "$out" \
|
|
> "$log" 2>&1
|
|
local t1; t1=$(date +%s)
|
|
# authoritative re-scored native fail count
|
|
local line fails evals topos
|
|
line=$(grep 're-scored (native)' "$log" | tail -1)
|
|
fails=$(printf '%s' "$line" | sed -n 's/.*(\([0-9]*\) fails).*/\1/p')
|
|
evals=$(grep -m1 '^evals' "$log" | sed -n 's/.*: \([0-9]*\) across.*/\1/p')
|
|
topos=$(grep -m1 '^evals' "$log" | sed -n 's/.*across \([0-9]*\) topologies.*/\1/p')
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
|
"$prog" "$seed" "$cfg" "${fails:-ERR}" "${evals:-?}" "${topos:-?}" "$((t1-t0))" >> "$TSV"
|
|
echo " -> ${fails:-ERR} fails, ${topos:-?} topologies, $((t1-t0))s"
|
|
}
|
|
|
|
for prog in maple-court harbor-house; do
|
|
ms="${MAXSHAPE[$prog]}"
|
|
for seed in "${SEEDS[@]}"; do
|
|
run "$prog" "$seed" baseline REASSOC=0 FEAS=0
|
|
run "$prog" "$seed" reassoc REASSOC=1 FEAS=0
|
|
run "$prog" "$seed" feas REASSOC=0 FEAS=1 MAXSHAPE="$ms"
|
|
run "$prog" "$seed" combined REASSOC=1 FEAS=1 MAXSHAPE="$ms"
|
|
done
|
|
done
|
|
|
|
echo "=== sweep complete ==="
|
|
column -t -s $'\t' "$TSV"
|