From 8486331fd459bb101867c6296926a0a6263587d3 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Mon, 27 Jul 2026 11:28:12 +0100 Subject: [PATCH 1/3] feat(clean_slate): remove auto-simulated datasets by simulator provenance clean_slate.sh restored modified datasets but never reclaimed the simulated ones, so every workspace accumulated regenerable dataset/ payload indefinitely. Deleting "everything untracked under dataset/" would be wrong. Each workspace ignores dataset/ wholesale and force-adds real datasets back, so untracked does not imply regenerable: SMACS J0723 (52 MB) and a2744/data.fits are real data downloaded at runtime rather than redistributed -- the Mahler et al. Lenstool files are GPL-licensed and deliberately not committed. No intrinsic marker separates the two kinds either; README.md and tracer.json occur in both. So a dataset is removed only when a simulator script in the same repo demonstrably writes it: both its type and name appear as string literals in one scripts/**/simulator*.py or scripts/**/simulators/*.py. Everything else is kept, so the rule errs toward keeping (start_here-generated datasets survive). Scope is the 4 workspaces + 3 HowTo repos; autolens_profiling and autolens_jax_joss are excluded because their dataset/ dirs hold real instrument data (alma/sma/hst inputs, JWST cosmos_web_ring). Only directories are candidates -- loose untracked files inside a committed dataset (a2744/data.fits, double_einstein_ring/*.png) have generic names that would match almost any simulator. Also warns when a committed dataset file exceeds 5 MB, as a tripwire against large real data being added to a workspace. Verified: 88 paths removed across 7 repos (~13 MB); zero tracked files in the delete list; all 9 known-real datasets intact; all 7 repos report 0 dirty files afterwards, confirming only ignored paths were touched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012EZTtyLUAyKWuATyk4mkcm --- bin/clean_slate.sh | 104 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/bin/clean_slate.sh b/bin/clean_slate.sh index dd19841..ec47451 100755 --- a/bin/clean_slate.sh +++ b/bin/clean_slate.sh @@ -7,6 +7,8 @@ # 1. restores tracked files under any dataset/ dir that a run modified in place # — the shipped datasets (cosmos_web_ring, simple, …) go back to their # committed state; they are never deleted. +# 1b. removes AUTO-SIMULATED datasets from the workspace/tutorial repos only +# (see DATASET_REPOS), and warns about oversized committed datasets. # 2. clears every output/ and scratch/ directory (model fits, scratch space). # 3. removes generated test_report.md files. # 4. removes ignored, fully-untracked top-level *.egg-info/ and build/ @@ -17,6 +19,24 @@ # - outside output/ and scratch/, it removes only the exact ignored top-level # packaging names above, and skips a candidate if it contains tracked files. # +# __Why datasets need provenance, not just "is it committed?"__ +# +# Every workspace ignores dataset/ wholesale and force-adds the real datasets +# back, so "untracked" does NOT mean "regenerable": some real datasets are +# deliberately never committed because they are DOWNLOADED at runtime rather +# than redistributed (autolens_workspace scripts/cluster/lenstool/data.py fetches +# SMACS J0723 from the GPL-licensed Mahler et al. repo + the STScI RELICS +# archive; dataset/cluster/a2744/data.fits is a hips2fits cutout). Deleting +# those costs a large re-download, and committing them would be a redistribution +# problem. No intrinsic marker separates them from simulated data either — a +# README or a tracer.json sits in both kinds. +# +# So a dataset is removed only when a simulator script in the SAME repo demonstrably +# writes it: both its dataset type and its name appear as string literals in one +# scripts/**/simulator*.py or scripts/**/simulators/*.py. Anything with no such +# provenance is kept. The rule therefore errs toward keeping — datasets written by +# start_here.py-style scripts survive, which is the safe direction. +# # Workspace root: PYAUTO_ROOT (default ~/Code/PyAutoLabs). # Preview without changing anything: DRY_RUN=1 clean_slate.sh # Packaging products only: clean_slate.sh --packaging @@ -28,6 +48,13 @@ DRY_RUN="${DRY_RUN:-0}" tag=""; [ "$DRY_RUN" = 1 ] && tag="[dry-run] " SCOPE="all" PACKAGING_REPOS=(PyAutoNerves PyAutoFit PyAutoArray PyAutoGalaxy PyAutoLens) +# Repos whose dataset/ dirs are auto-simulated by their own simulator scripts. +# Deliberately excludes autolens_profiling and autolens_jax_joss, whose dataset/ +# dirs hold real instrument data (alma/sma/hst inputs, JWST cosmos_web_ring). +DATASET_REPOS=(autolens_workspace autogalaxy_workspace autofit_workspace \ + autocti_workspace HowToLens HowToGalaxy HowToFit) +# Committed dataset files above this size are flagged as repo bloat. +DATASET_WARN_KB=5120 case "${1:-}" in --packaging) SCOPE="packaging" ;; "") ;; @@ -42,12 +69,70 @@ is_packaging_repo() { return 1 } +is_dataset_repo() { + local candidate + for candidate in "${DATASET_REPOS[@]}"; do + [ "$candidate" = "$1" ] && return 0 + done + return 1 +} + +# Print each untracked path under dataset/ that a simulator script in this repo +# writes. Expands git-clean's collapsed entries down to dataset// +# granularity so a wholly-untracked dataset/ tree is still judged per dataset. +simulated_datasets() { + local repo="$1" path depth child type name matched + local -a sims queue found + mapfile -t sims < <(find "$repo/scripts" -type f -name '*.py' \ + \( -name 'simulator*' -o -path '*/simulators/*' \) 2>/dev/null) + [ "${#sims[@]}" -eq 0 ] && return 0 + + queue=() + while IFS= read -r path; do + queue+=("${path#Would remove }") + done < <(git -C "$repo" clean -ndx -- dataset 2>/dev/null) + + found=() + while [ "${#queue[@]}" -gt 0 ]; do + path="${queue[0]%/}"; queue=("${queue[@]:1}") + depth=$(awk -F/ '{print NF}' <<<"$path") + if [ -d "$repo/$path" ] && [ "$depth" -lt 3 ]; then + while IFS= read -r child; do + queue+=("${child#"$repo"/}") + done < <(find "$repo/$path" -mindepth 1 -maxdepth 1 2>/dev/null) + continue + fi + # A dataset is a DIRECTORY. Loose untracked files are never candidates: + # they are usually byproducts sitting inside a committed real dataset + # (dataset/cluster/a2744/data.fits — a downloaded HST cutout re-ignored + # by .gitignore; double_einstein_ring/*.png), and their generic names + # ("data.fits") would match almost any simulator script. + [ -d "$repo/$path" ] || continue + # Defence in depth: never touch a directory holding a tracked file. + # (Do NOT extend this to the parent — a type directory like + # dataset/imaging/ legitimately holds committed datasets alongside + # simulated ones, so a parent test would protect everything.) + [ -z "$(git -C "$repo" ls-files -- "$path" 2>/dev/null)" ] || continue + type=$(awk -F/ '{print $2}' <<<"$path") + name="${path##*/}" + matched="" + while IFS= read -r hit; do + grep -qF "\"$type\"" "$hit" 2>/dev/null && { matched=1; break; } + done < <(grep -lF "\"$name\"" "${sims[@]}" 2>/dev/null) + [ -n "$matched" ] && found+=("$path") + done + [ "${#found[@]}" -gt 0 ] && printf '%s\n' "${found[@]}" + return 0 +} + for dir in */; do repo="${dir%/}" [ -d "$repo/.git" ] || continue header="" show() { [ -z "$header" ] && { echo "=== $repo ==="; header=1; }; echo " ${tag}$*"; } + # Observations, not pending actions — never carry the [dry-run] tag. + warn() { [ -z "$header" ] && { echo "=== $repo ==="; header=1; }; echo " $*"; } if [ "$SCOPE" = all ]; then # 1. Restore shipped datasets modified in place. @@ -57,6 +142,25 @@ for dir in */; do [ "$DRY_RUN" = 1 ] || printf '%s\0' "${moddata[@]}" | xargs -0 -r git -C "$repo" checkout -- fi + # 1b. Remove auto-simulated datasets, and flag oversized committed ones. + if is_dataset_repo "$repo"; then + nsim=0; simkb=0 + while IFS= read -r rel; do + [ -n "$rel" ] || continue + nsim=$((nsim + 1)) + kb=$(du -sk "$repo/$rel" 2>/dev/null | cut -f1) + simkb=$((simkb + ${kb:-0})) + [ "$DRY_RUN" = 1 ] || rm -rf "${repo:?}/${rel:?}" + done < <(simulated_datasets "$repo") + [ "$nsim" -gt 0 ] && show "remove $nsim simulated dataset(s) ($((simkb / 1024)) MB)" + + while IFS= read -r -d '' f; do + kb=$(du -sk "$repo/$f" 2>/dev/null | cut -f1) + [ "${kb:-0}" -gt "$DATASET_WARN_KB" ] || continue + warn "WARNING: committed dataset $f is $((kb / 1024)) MB (>$((DATASET_WARN_KB / 1024)) MB)" + done < <(git -C "$repo" ls-files -z -- 'dataset/*' 2>/dev/null) + fi + # 2. Clear output/ and scratch/ dirs (untracked + ignored inside them; tracked kept). while IFS= read -r -d '' d; do rel="${d#"$repo"/}" From 10d0708816d11be200e0516541ae66761bdcb38c Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Mon, 27 Jul 2026 11:32:38 +0100 Subject: [PATCH 2/3] fix(clean_slate): match scripts/simulator/ (singular) as well as simulators/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The simulator-script glob matched files named simulator*.py or any file under a directory named simulators/ (plural). HowToLens keeps its five simulators in scripts/simulator/ (singular), which matches neither — so the repo resolved to zero simulator scripts and the dataset sweep silently did nothing there. Surveyed all seven in-scope repos: HowToLens is the only one affected, and it is now covered (5 of its 6 dataset dirs removed; dataset/imaging/howtolens is written by the tutorial chapters rather than a simulator, so it is correctly kept). A missing spelling disables a whole repo without any error, so the glob now lists both directory forms explicitly and says why. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012EZTtyLUAyKWuATyk4mkcm --- bin/clean_slate.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bin/clean_slate.sh b/bin/clean_slate.sh index ec47451..15f36a4 100755 --- a/bin/clean_slate.sh +++ b/bin/clean_slate.sh @@ -31,11 +31,12 @@ # problem. No intrinsic marker separates them from simulated data either — a # README or a tracer.json sits in both kinds. # -# So a dataset is removed only when a simulator script in the SAME repo demonstrably -# writes it: both its dataset type and its name appear as string literals in one -# scripts/**/simulator*.py or scripts/**/simulators/*.py. Anything with no such -# provenance is kept. The rule therefore errs toward keeping — datasets written by -# start_here.py-style scripts survive, which is the safe direction. +# So a dataset is removed only when a simulator script in the SAME repo +# demonstrably writes it: both its dataset type and its name appear as string +# literals in one scripts/**/simulator*.py, scripts/**/simulator/*.py or +# scripts/**/simulators/*.py. Anything with no such provenance is kept. The rule +# therefore errs toward keeping — datasets written by start_here.py-style scripts +# survive, which is the safe direction. # # Workspace root: PYAUTO_ROOT (default ~/Code/PyAutoLabs). # Preview without changing anything: DRY_RUN=1 clean_slate.sh @@ -83,8 +84,11 @@ is_dataset_repo() { simulated_datasets() { local repo="$1" path depth child type name matched local -a sims queue found + # Both directory spellings are in use: HowToLens uses scripts/simulator/ + # (singular), everything else scripts/**/simulators/ (plural) or a + # simulator*.py filename. Missing one silently disables a whole repo. mapfile -t sims < <(find "$repo/scripts" -type f -name '*.py' \ - \( -name 'simulator*' -o -path '*/simulators/*' \) 2>/dev/null) + \( -name 'simulator*' -o -path '*/simulator/*' -o -path '*/simulators/*' \) 2>/dev/null) [ "${#sims[@]}" -eq 0 ] && return 0 queue=() From 74e87496dd4595412aad2894d054a95595b93ea4 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Mon, 27 Jul 2026 11:37:56 +0100 Subject: [PATCH 3/3] feat(clean_slate): warn when an in-scope repo yields no simulator scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "No simulator scripts found" and "this repo has nothing to clean" produced identical output — an empty result and a silent no-op. That is how HowToLens's scripts/simulator/ (singular) went unnoticed: the sweep reported nothing and looked correct. An in-scope repo with zero simulator scripts is almost certainly an unrecognised layout, so say so instead of no-opping quietly. Extracted the glob into simulator_scripts() so the sweep and the check share one definition and cannot drift apart. Verified by adding a simulator-less repo to DATASET_REPOS in a scratch copy: the warning fires. It stays silent for all seven real in-scope repos. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012EZTtyLUAyKWuATyk4mkcm --- bin/clean_slate.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/bin/clean_slate.sh b/bin/clean_slate.sh index 15f36a4..248e37d 100755 --- a/bin/clean_slate.sh +++ b/bin/clean_slate.sh @@ -81,14 +81,19 @@ is_dataset_repo() { # Print each untracked path under dataset/ that a simulator script in this repo # writes. Expands git-clean's collapsed entries down to dataset// # granularity so a wholly-untracked dataset/ tree is still judged per dataset. +# Both directory spellings are in use: HowToLens uses scripts/simulator/ +# (singular), everything else scripts/**/simulators/ (plural) or a simulator*.py +# filename. A spelling this misses yields zero scripts, which is indistinguishable +# from "nothing to clean" — hence the empty-set warning at the call site. +simulator_scripts() { + find "$1/scripts" -type f -name '*.py' \ + \( -name 'simulator*' -o -path '*/simulator/*' -o -path '*/simulators/*' \) 2>/dev/null +} + simulated_datasets() { local repo="$1" path depth child type name matched local -a sims queue found - # Both directory spellings are in use: HowToLens uses scripts/simulator/ - # (singular), everything else scripts/**/simulators/ (plural) or a - # simulator*.py filename. Missing one silently disables a whole repo. - mapfile -t sims < <(find "$repo/scripts" -type f -name '*.py' \ - \( -name 'simulator*' -o -path '*/simulator/*' -o -path '*/simulators/*' \) 2>/dev/null) + mapfile -t sims < <(simulator_scripts "$repo") [ "${#sims[@]}" -eq 0 ] && return 0 queue=() @@ -148,6 +153,13 @@ for dir in */; do # 1b. Remove auto-simulated datasets, and flag oversized committed ones. if is_dataset_repo "$repo"; then + # An in-scope repo with no simulator scripts is almost certainly an + # unrecognised layout, not a repo with nothing to simulate — the two + # are otherwise indistinguishable (this is how scripts/simulator/ + # went unnoticed in HowToLens). Say so rather than no-op in silence. + if [ "$(simulator_scripts "$repo" | wc -l)" -eq 0 ]; then + warn "WARNING: no simulator scripts found — dataset sweep disabled for this repo" + fi nsim=0; simkb=0 while IFS= read -r rel; do [ -n "$rel" ] || continue