feat(clean_slate): remove auto-simulated datasets by simulator provenance

bin/clean_slate.sh

@@ -7,6 +7,8 @@
 #   1. restores tracked files under any dataset/ dir that a run modified in place
 #      — the shipped datasets (cosmos_web_ring, simple, …) go back to their
 #      committed state; they are never deleted.
+#   1b. removes AUTO-SIMULATED datasets from the workspace/tutorial repos only
+#      (see DATASET_REPOS), and warns about oversized committed datasets.
 #   2. clears every output/ and scratch/ directory (model fits, scratch space).
 #   3. removes generated test_report.md files.
 #   4. removes ignored, fully-untracked top-level *.egg-info/ and build/
@@ -17,6 +19,25 @@
 #   - outside output/ and scratch/, it removes only the exact ignored top-level
 #     packaging names above, and skips a candidate if it contains tracked files.
 #
+# __Why datasets need provenance, not just "is it committed?"__
+#
+# Every workspace ignores dataset/ wholesale and force-adds the real datasets
+# back, so "untracked" does NOT mean "regenerable": some real datasets are
+# deliberately never committed because they are DOWNLOADED at runtime rather
+# than redistributed (autolens_workspace scripts/cluster/lenstool/data.py fetches
+# SMACS J0723 from the GPL-licensed Mahler et al. repo + the STScI RELICS
+# archive; dataset/cluster/a2744/data.fits is a hips2fits cutout). Deleting
+# those costs a large re-download, and committing them would be a redistribution
+# problem. No intrinsic marker separates them from simulated data either — a
+# README or a tracer.json sits in both kinds.
+#
+# So a dataset is removed only when a simulator script in the SAME repo
+# demonstrably writes it: both its dataset type and its name appear as string
+# literals in one scripts/**/simulator*.py, scripts/**/simulator/*.py or
+# scripts/**/simulators/*.py. Anything with no such provenance is kept. The rule
+# therefore errs toward keeping — datasets written by start_here.py-style scripts
+# survive, which is the safe direction.
+#
 # Workspace root: PYAUTO_ROOT (default ~/Code/PyAutoLabs).
 # Preview without changing anything:  DRY_RUN=1 clean_slate.sh
 # Packaging products only:            clean_slate.sh --packaging
@@ -28,6 +49,13 @@ DRY_RUN="${DRY_RUN:-0}"
 tag=""; [ "$DRY_RUN" = 1 ] && tag="[dry-run] "
 SCOPE="all"
 PACKAGING_REPOS=(PyAutoNerves PyAutoFit PyAutoArray PyAutoGalaxy PyAutoLens)
+# Repos whose dataset/ dirs are auto-simulated by their own simulator scripts.
+# Deliberately excludes autolens_profiling and autolens_jax_joss, whose dataset/
+# dirs hold real instrument data (alma/sma/hst inputs, JWST cosmos_web_ring).
+DATASET_REPOS=(autolens_workspace autogalaxy_workspace autofit_workspace \
+               autocti_workspace HowToLens HowToGalaxy HowToFit)
+# Committed dataset files above this size are flagged as repo bloat.
+DATASET_WARN_KB=5120
 case "${1:-}" in
     --packaging) SCOPE="packaging" ;;
     "") ;;
@@ -42,12 +70,78 @@ is_packaging_repo() {
     return 1
 }
 
+is_dataset_repo() {
+    local candidate
+    for candidate in "${DATASET_REPOS[@]}"; do
+        [ "$candidate" = "$1" ] && return 0
+    done
+    return 1
+}
+
+# Print each untracked path under dataset/ that a simulator script in this repo
+# writes. Expands git-clean's collapsed entries down to dataset/<type>/<name>
+# granularity so a wholly-untracked dataset/ tree is still judged per dataset.
+# Both directory spellings are in use: HowToLens uses scripts/simulator/
+# (singular), everything else scripts/**/simulators/ (plural) or a simulator*.py
+# filename. A spelling this misses yields zero scripts, which is indistinguishable
+# from "nothing to clean" — hence the empty-set warning at the call site.
+simulator_scripts() {
+    find "$1/scripts" -type f -name '*.py' \
+        \( -name 'simulator*' -o -path '*/simulator/*' -o -path '*/simulators/*' \) 2>/dev/null
+}
+
+simulated_datasets() {
+    local repo="$1" path depth child type name matched
+    local -a sims queue found
+    mapfile -t sims < <(simulator_scripts "$repo")
+    [ "${#sims[@]}" -eq 0 ] && return 0
+
+    queue=()
+    while IFS= read -r path; do
+        queue+=("${path#Would remove }")
+    done < <(git -C "$repo" clean -ndx -- dataset 2>/dev/null)
+
+    found=()
+    while [ "${#queue[@]}" -gt 0 ]; do
+        path="${queue[0]%/}"; queue=("${queue[@]:1}")
+        depth=$(awk -F/ '{print NF}' <<<"$path")
+        if [ -d "$repo/$path" ] && [ "$depth" -lt 3 ]; then
+            while IFS= read -r child; do
+                queue+=("${child#"$repo"/}")
+            done < <(find "$repo/$path" -mindepth 1 -maxdepth 1 2>/dev/null)
+            continue
+        fi
+        # A dataset is a DIRECTORY. Loose untracked files are never candidates:
+        # they are usually byproducts sitting inside a committed real dataset
+        # (dataset/cluster/a2744/data.fits — a downloaded HST cutout re-ignored
+        # by .gitignore; double_einstein_ring/*.png), and their generic names
+        # ("data.fits") would match almost any simulator script.
+        [ -d "$repo/$path" ] || continue
+        # Defence in depth: never touch a directory holding a tracked file.
+        # (Do NOT extend this to the parent — a type directory like
+        # dataset/imaging/ legitimately holds committed datasets alongside
+        # simulated ones, so a parent test would protect everything.)
+        [ -z "$(git -C "$repo" ls-files -- "$path" 2>/dev/null)" ] || continue
+        type=$(awk -F/ '{print $2}' <<<"$path")
+        name="${path##*/}"
+        matched=""
+        while IFS= read -r hit; do
+            grep -qF "\"$type\"" "$hit" 2>/dev/null && { matched=1; break; }
+        done < <(grep -lF "\"$name\"" "${sims[@]}" 2>/dev/null)
+        [ -n "$matched" ] && found+=("$path")
+    done
+    [ "${#found[@]}" -gt 0 ] && printf '%s\n' "${found[@]}"
+    return 0
+}
+
 for dir in */; do
     repo="${dir%/}"
     [ -d "$repo/.git" ] || continue
     header=""
 
     show() { [ -z "$header" ] && { echo "=== $repo ==="; header=1; }; echo "  ${tag}$*"; }
+    # Observations, not pending actions — never carry the [dry-run] tag.
+    warn() { [ -z "$header" ] && { echo "=== $repo ==="; header=1; }; echo "  $*"; }
 
     if [ "$SCOPE" = all ]; then
         # 1. Restore shipped datasets modified in place.
@@ -57,6 +151,32 @@ for dir in */; do
             [ "$DRY_RUN" = 1 ] || printf '%s\0' "${moddata[@]}" | xargs -0 -r git -C "$repo" checkout --
         fi
 
+        # 1b. Remove auto-simulated datasets, and flag oversized committed ones.
+        if is_dataset_repo "$repo"; then
+            # An in-scope repo with no simulator scripts is almost certainly an
+            # unrecognised layout, not a repo with nothing to simulate — the two
+            # are otherwise indistinguishable (this is how scripts/simulator/
+            # went unnoticed in HowToLens). Say so rather than no-op in silence.
+            if [ "$(simulator_scripts "$repo" | wc -l)" -eq 0 ]; then
+                warn "WARNING: no simulator scripts found — dataset sweep disabled for this repo"
+            fi
+            nsim=0; simkb=0
+            while IFS= read -r rel; do
+                [ -n "$rel" ] || continue
+                nsim=$((nsim + 1))
+                kb=$(du -sk "$repo/$rel" 2>/dev/null | cut -f1)
+                simkb=$((simkb + ${kb:-0}))
+                [ "$DRY_RUN" = 1 ] || rm -rf "${repo:?}/${rel:?}"
+            done < <(simulated_datasets "$repo")
+            [ "$nsim" -gt 0 ] && show "remove $nsim simulated dataset(s) ($((simkb / 1024)) MB)"
+
+            while IFS= read -r -d '' f; do
+                kb=$(du -sk "$repo/$f" 2>/dev/null | cut -f1)
+                [ "${kb:-0}" -gt "$DATASET_WARN_KB" ] || continue
+                warn "WARNING: committed dataset $f is $((kb / 1024)) MB (>$((DATASET_WARN_KB / 1024)) MB)"
+            done < <(git -C "$repo" ls-files -z -- 'dataset/*' 2>/dev/null)
+        fi
+
         # 2. Clear output/ and scratch/ dirs (untracked + ignored inside them; tracked kept).
         while IFS= read -r -d '' d; do
             rel="${d#"$repo"/}"