diff --git a/bin/clean_slate.sh b/bin/clean_slate.sh index dd19841..248e37d 100755 --- a/bin/clean_slate.sh +++ b/bin/clean_slate.sh @@ -7,6 +7,8 @@ # 1. restores tracked files under any dataset/ dir that a run modified in place # — the shipped datasets (cosmos_web_ring, simple, …) go back to their # committed state; they are never deleted. +# 1b. removes AUTO-SIMULATED datasets from the workspace/tutorial repos only +# (see DATASET_REPOS), and warns about oversized committed datasets. # 2. clears every output/ and scratch/ directory (model fits, scratch space). # 3. removes generated test_report.md files. # 4. removes ignored, fully-untracked top-level *.egg-info/ and build/ @@ -17,6 +19,25 @@ # - outside output/ and scratch/, it removes only the exact ignored top-level # packaging names above, and skips a candidate if it contains tracked files. # +# __Why datasets need provenance, not just "is it committed?"__ +# +# Every workspace ignores dataset/ wholesale and force-adds the real datasets +# back, so "untracked" does NOT mean "regenerable": some real datasets are +# deliberately never committed because they are DOWNLOADED at runtime rather +# than redistributed (autolens_workspace scripts/cluster/lenstool/data.py fetches +# SMACS J0723 from the GPL-licensed Mahler et al. repo + the STScI RELICS +# archive; dataset/cluster/a2744/data.fits is a hips2fits cutout). Deleting +# those costs a large re-download, and committing them would be a redistribution +# problem. No intrinsic marker separates them from simulated data either — a +# README or a tracer.json sits in both kinds. +# +# So a dataset is removed only when a simulator script in the SAME repo +# demonstrably writes it: both its dataset type and its name appear as string +# literals in one scripts/**/simulator*.py, scripts/**/simulator/*.py or +# scripts/**/simulators/*.py. Anything with no such provenance is kept. The rule +# therefore errs toward keeping — datasets written by start_here.py-style scripts +# survive, which is the safe direction. +# # Workspace root: PYAUTO_ROOT (default ~/Code/PyAutoLabs). # Preview without changing anything: DRY_RUN=1 clean_slate.sh # Packaging products only: clean_slate.sh --packaging @@ -28,6 +49,13 @@ DRY_RUN="${DRY_RUN:-0}" tag=""; [ "$DRY_RUN" = 1 ] && tag="[dry-run] " SCOPE="all" PACKAGING_REPOS=(PyAutoNerves PyAutoFit PyAutoArray PyAutoGalaxy PyAutoLens) +# Repos whose dataset/ dirs are auto-simulated by their own simulator scripts. +# Deliberately excludes autolens_profiling and autolens_jax_joss, whose dataset/ +# dirs hold real instrument data (alma/sma/hst inputs, JWST cosmos_web_ring). +DATASET_REPOS=(autolens_workspace autogalaxy_workspace autofit_workspace \ + autocti_workspace HowToLens HowToGalaxy HowToFit) +# Committed dataset files above this size are flagged as repo bloat. +DATASET_WARN_KB=5120 case "${1:-}" in --packaging) SCOPE="packaging" ;; "") ;; @@ -42,12 +70,78 @@ is_packaging_repo() { return 1 } +is_dataset_repo() { + local candidate + for candidate in "${DATASET_REPOS[@]}"; do + [ "$candidate" = "$1" ] && return 0 + done + return 1 +} + +# Print each untracked path under dataset/ that a simulator script in this repo +# writes. Expands git-clean's collapsed entries down to dataset// +# granularity so a wholly-untracked dataset/ tree is still judged per dataset. +# Both directory spellings are in use: HowToLens uses scripts/simulator/ +# (singular), everything else scripts/**/simulators/ (plural) or a simulator*.py +# filename. A spelling this misses yields zero scripts, which is indistinguishable +# from "nothing to clean" — hence the empty-set warning at the call site. +simulator_scripts() { + find "$1/scripts" -type f -name '*.py' \ + \( -name 'simulator*' -o -path '*/simulator/*' -o -path '*/simulators/*' \) 2>/dev/null +} + +simulated_datasets() { + local repo="$1" path depth child type name matched + local -a sims queue found + mapfile -t sims < <(simulator_scripts "$repo") + [ "${#sims[@]}" -eq 0 ] && return 0 + + queue=() + while IFS= read -r path; do + queue+=("${path#Would remove }") + done < <(git -C "$repo" clean -ndx -- dataset 2>/dev/null) + + found=() + while [ "${#queue[@]}" -gt 0 ]; do + path="${queue[0]%/}"; queue=("${queue[@]:1}") + depth=$(awk -F/ '{print NF}' <<<"$path") + if [ -d "$repo/$path" ] && [ "$depth" -lt 3 ]; then + while IFS= read -r child; do + queue+=("${child#"$repo"/}") + done < <(find "$repo/$path" -mindepth 1 -maxdepth 1 2>/dev/null) + continue + fi + # A dataset is a DIRECTORY. Loose untracked files are never candidates: + # they are usually byproducts sitting inside a committed real dataset + # (dataset/cluster/a2744/data.fits — a downloaded HST cutout re-ignored + # by .gitignore; double_einstein_ring/*.png), and their generic names + # ("data.fits") would match almost any simulator script. + [ -d "$repo/$path" ] || continue + # Defence in depth: never touch a directory holding a tracked file. + # (Do NOT extend this to the parent — a type directory like + # dataset/imaging/ legitimately holds committed datasets alongside + # simulated ones, so a parent test would protect everything.) + [ -z "$(git -C "$repo" ls-files -- "$path" 2>/dev/null)" ] || continue + type=$(awk -F/ '{print $2}' <<<"$path") + name="${path##*/}" + matched="" + while IFS= read -r hit; do + grep -qF "\"$type\"" "$hit" 2>/dev/null && { matched=1; break; } + done < <(grep -lF "\"$name\"" "${sims[@]}" 2>/dev/null) + [ -n "$matched" ] && found+=("$path") + done + [ "${#found[@]}" -gt 0 ] && printf '%s\n' "${found[@]}" + return 0 +} + for dir in */; do repo="${dir%/}" [ -d "$repo/.git" ] || continue header="" show() { [ -z "$header" ] && { echo "=== $repo ==="; header=1; }; echo " ${tag}$*"; } + # Observations, not pending actions — never carry the [dry-run] tag. + warn() { [ -z "$header" ] && { echo "=== $repo ==="; header=1; }; echo " $*"; } if [ "$SCOPE" = all ]; then # 1. Restore shipped datasets modified in place. @@ -57,6 +151,32 @@ for dir in */; do [ "$DRY_RUN" = 1 ] || printf '%s\0' "${moddata[@]}" | xargs -0 -r git -C "$repo" checkout -- fi + # 1b. Remove auto-simulated datasets, and flag oversized committed ones. + if is_dataset_repo "$repo"; then + # An in-scope repo with no simulator scripts is almost certainly an + # unrecognised layout, not a repo with nothing to simulate — the two + # are otherwise indistinguishable (this is how scripts/simulator/ + # went unnoticed in HowToLens). Say so rather than no-op in silence. + if [ "$(simulator_scripts "$repo" | wc -l)" -eq 0 ]; then + warn "WARNING: no simulator scripts found — dataset sweep disabled for this repo" + fi + nsim=0; simkb=0 + while IFS= read -r rel; do + [ -n "$rel" ] || continue + nsim=$((nsim + 1)) + kb=$(du -sk "$repo/$rel" 2>/dev/null | cut -f1) + simkb=$((simkb + ${kb:-0})) + [ "$DRY_RUN" = 1 ] || rm -rf "${repo:?}/${rel:?}" + done < <(simulated_datasets "$repo") + [ "$nsim" -gt 0 ] && show "remove $nsim simulated dataset(s) ($((simkb / 1024)) MB)" + + while IFS= read -r -d '' f; do + kb=$(du -sk "$repo/$f" 2>/dev/null | cut -f1) + [ "${kb:-0}" -gt "$DATASET_WARN_KB" ] || continue + warn "WARNING: committed dataset $f is $((kb / 1024)) MB (>$((DATASET_WARN_KB / 1024)) MB)" + done < <(git -C "$repo" ls-files -z -- 'dataset/*' 2>/dev/null) + fi + # 2. Clear output/ and scratch/ dirs (untracked + ignored inside them; tracked kept). while IFS= read -r -d '' d; do rel="${d#"$repo"/}"