diff --git a/bin/clean_slate.sh b/bin/clean_slate.sh index 248e37d..59bb016 100755 --- a/bin/clean_slate.sh +++ b/bin/clean_slate.sh @@ -7,12 +7,15 @@ # 1. restores tracked files under any dataset/ dir that a run modified in place # — the shipped datasets (cosmos_web_ring, simple, …) go back to their # committed state; they are never deleted. -# 1b. removes AUTO-SIMULATED datasets from the workspace/tutorial repos only -# (see DATASET_REPOS), and warns about oversized committed datasets. +# 1b. removes REGENERABLE datasets from the workspace/tutorial repos only +# (see DATASET_REPOS), reports orphans, prunes emptied dataset dirs, and +# warns about oversized committed datasets. # 2. clears every output/ and scratch/ directory (model fits, scratch space). -# 3. removes generated test_report.md files. +# 3. removes generated test_report.md files, then untracked .ipynb_checkpoints/ +# directories repo-wide. # 4. removes ignored, fully-untracked top-level *.egg-info/ and build/ # packaging directories from the managed library repos. +# 5. runs `git gc --auto` per repo to compact the object store. # # It is git-aware and conservative: # - it never deletes a tracked file (except reverting in-place dataset edits); @@ -31,18 +34,32 @@ # problem. No intrinsic marker separates them from simulated data either — a # README or a tracer.json sits in both kinds. # -# So a dataset is removed only when a simulator script in the SAME repo -# demonstrably writes it: both its dataset type and its name appear as string -# literals in one scripts/**/simulator*.py, scripts/**/simulator/*.py or -# scripts/**/simulators/*.py. Anything with no such provenance is kept. The rule -# therefore errs toward keeping — datasets written by start_here.py-style scripts -# survive, which is the safe direction. +# __The write-site rule__ +# +# A dataset is removed only when a script in the SAME repo demonstrably WRITES +# it — a name mention is never enough. bin/dataset_provenance.py parses every +# scripts/**/*.py with `ast` and, in source order, tracks which dataset path each +# variable currently holds and whether that variable reaches an output call +# (output_to_fits, fits_imaging, json.dump, open(..., "w"), a helper whose body +# writes the parameter, …). That order sensitivity is the point: +# scripts/interferometer/start_here.py binds `dataset_path` to the real sdp81 +# data (which it only reads), then rebinds it to simulated_lens (which it +# writes) — a grep for the dataset name cannot tell those apart, and the old +# name-literal rule missed every dataset written by a start_here.py (#167). +# +# Three verdicts come back per candidate: +# REGENERABLE positive write evidence, no network — deleted. +# DOWNLOADED its binding script also fetches over the network — kept +# silently; this is real data cached rather than redistributed. +# ORPHAN no writer found — kept and REPORTED, so a human can look. +# Deletion requires positive evidence; every uncertainty lands on ORPHAN. # # Workspace root: PYAUTO_ROOT (default ~/Code/PyAutoLabs). # Preview without changing anything: DRY_RUN=1 clean_slate.sh # Packaging products only: clean_slate.sh --packaging set -u +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" ROOT="${PYAUTO_ROOT:-$HOME/Code/PyAutoLabs}" cd "$ROOT" || { echo "workspace root not found: $ROOT" >&2; exit 1; } DRY_RUN="${DRY_RUN:-0}" @@ -78,23 +95,13 @@ is_dataset_repo() { return 1 } -# Print each untracked path under dataset/ that a simulator script in this repo -# writes. Expands git-clean's collapsed entries down to dataset// -# granularity so a wholly-untracked dataset/ tree is still judged per dataset. -# Both directory spellings are in use: HowToLens uses scripts/simulator/ -# (singular), everything else scripts/**/simulators/ (plural) or a simulator*.py -# filename. A spelling this misses yields zero scripts, which is indistinguishable -# from "nothing to clean" — hence the empty-set warning at the call site. -simulator_scripts() { - find "$1/scripts" -type f -name '*.py' \ - \( -name 'simulator*' -o -path '*/simulator/*' -o -path '*/simulators/*' \) 2>/dev/null -} - -simulated_datasets() { - local repo="$1" path depth child type name matched - local -a sims queue found - mapfile -t sims < <(simulator_scripts "$repo") - [ "${#sims[@]}" -eq 0 ] && return 0 +# Print every untracked dataset// DIRECTORY worth judging. Expands +# git-clean's collapsed entries down to dataset// granularity so a +# wholly-untracked dataset/ tree is still judged per dataset. Provenance itself +# is decided by dataset_provenance.py — this only produces the candidate list. +dataset_candidates() { + local repo="$1" path depth child + local -a queue found queue=() while IFS= read -r path; do @@ -122,13 +129,7 @@ simulated_datasets() { # dataset/imaging/ legitimately holds committed datasets alongside # simulated ones, so a parent test would protect everything.) [ -z "$(git -C "$repo" ls-files -- "$path" 2>/dev/null)" ] || continue - type=$(awk -F/ '{print $2}' <<<"$path") - name="${path##*/}" - matched="" - while IFS= read -r hit; do - grep -qF "\"$type\"" "$hit" 2>/dev/null && { matched=1; break; } - done < <(grep -lF "\"$name\"" "${sims[@]}" 2>/dev/null) - [ -n "$matched" ] && found+=("$path") + found+=("$path") done [ "${#found[@]}" -gt 0 ] && printf '%s\n' "${found[@]}" return 0 @@ -151,30 +152,70 @@ for dir in */; do [ "$DRY_RUN" = 1 ] || printf '%s\0' "${moddata[@]}" | xargs -0 -r git -C "$repo" checkout -- fi - # 1b. Remove auto-simulated datasets, and flag oversized committed ones. + # 1b. Classify untracked datasets by write site, remove the regenerable + # ones, report orphans, and flag oversized committed ones. if is_dataset_repo "$repo"; then - # An in-scope repo with no simulator scripts is almost certainly an - # unrecognised layout, not a repo with nothing to simulate — the two - # are otherwise indistinguishable (this is how scripts/simulator/ - # went unnoticed in HowToLens). Say so rather than no-op in silence. - if [ "$(simulator_scripts "$repo" | wc -l)" -eq 0 ]; then - warn "WARNING: no simulator scripts found — dataset sweep disabled for this repo" + mapfile -t candidates < <(dataset_candidates "$repo") + if [ "${#candidates[@]}" -gt 0 ]; then + errfile=$(mktemp) + verdicts=$(python3 "$SCRIPT_DIR/dataset_provenance.py" \ + --repo "$repo" "${candidates[@]}" 2>"$errfile") + status=$? + # No fallback: guessing provenance is exactly what this replaced. + if [ "$status" -ne 0 ]; then + cat "$errfile" >&2 + rm -f "$errfile" + echo "clean_slate: dataset_provenance.py failed for $repo (exit $status)" >&2 + exit 1 + fi + [ -s "$errfile" ] && cat "$errfile" >&2 + rm -f "$errfile" + + nsim=0; simkb=0 + while read -r verdict rel; do + [ -n "$rel" ] || continue + case "$verdict" in + REGENERABLE) + nsim=$((nsim + 1)) + kb=$(du -sk "$repo/$rel" 2>/dev/null | cut -f1) + simkb=$((simkb + ${kb:-0})) + [ "$DRY_RUN" = 1 ] || rm -rf "${repo:?}/${rel:?}" + ;; + ORPHAN) + # Never deleted — surfaced so a human can decide + # whether it is real data or forgotten cruft. + warn "orphan dataset (no writer): $rel ($(du -sh "$repo/$rel" 2>/dev/null | cut -f1))" + ;; + DOWNLOADED) ;; # real data, cached not redistributed — keep, silently + esac + done <<<"$verdicts" + [ "$nsim" -gt 0 ] && show "remove $nsim simulated dataset(s) ($((simkb / 1024)) MB)" fi - nsim=0; simkb=0 - while IFS= read -r rel; do - [ -n "$rel" ] || continue - nsim=$((nsim + 1)) - kb=$(du -sk "$repo/$rel" 2>/dev/null | cut -f1) - simkb=$((simkb + ${kb:-0})) - [ "$DRY_RUN" = 1 ] || rm -rf "${repo:?}/${rel:?}" - done < <(simulated_datasets "$repo") - [ "$nsim" -gt 0 ] && show "remove $nsim simulated dataset(s) ($((simkb / 1024)) MB)" - - while IFS= read -r -d '' f; do - kb=$(du -sk "$repo/$f" 2>/dev/null | cut -f1) - [ "${kb:-0}" -gt "$DATASET_WARN_KB" ] || continue - warn "WARNING: committed dataset $f is $((kb / 1024)) MB (>$((DATASET_WARN_KB / 1024)) MB)" - done < <(git -C "$repo" ls-files -z -- 'dataset/*' 2>/dev/null) + + # Prune directories the sweep just emptied (and any that were + # already empty). An empty directory is never tracked by git, so + # this can never touch committed content. + nempty=$(find "$repo/dataset" -mindepth 1 -type d -empty 2>/dev/null | wc -l) + if [ "$nempty" -gt 0 ]; then + show "remove $nempty empty dataset director$([ "$nempty" -eq 1 ] && echo y || echo ies)" + [ "$DRY_RUN" = 1 ] || find "$repo/dataset" -mindepth 1 -type d -empty -delete 2>/dev/null + fi + + # Bloat warning aggregated per dataset DIRECTORY, not per file: a + # dataset is many .fits files and one line per file buries the + # signal. Tracked bytes only — untracked cruft is not repo bloat. + while IFS=$'\t' read -r dir kb; do + warn "WARNING: committed dataset $dir is $((kb / 1024)) MB (>$((DATASET_WARN_KB / 1024)) MB)" + done < <(git -C "$repo" ls-files -z -- 'dataset/*' 2>/dev/null \ + | (cd "$repo" && xargs -0 -r du -k --apparent-size --) 2>/dev/null \ + | awk -F'\t' -v warn="$DATASET_WARN_KB" ' + NF == 2 { + n = split($2, part, "/") + dir = (n >= 3) ? part[1] "/" part[2] "/" part[3] : $2 + total[dir] += $1 + } + END { for (d in total) if (total[d] > warn) printf "%s\t%d\n", d, total[d] }' \ + | sort) fi # 2. Clear output/ and scratch/ dirs (untracked + ignored inside them; tracked kept). @@ -193,23 +234,43 @@ for dir in */; do show "remove $rel" [ "$DRY_RUN" = 1 ] || rm -f "$f" done < <(find "$repo" -maxdepth 2 -type f -name test_report.md -not -path '*/.git/*' -print0 2>/dev/null) + + # 3b. Remove untracked .ipynb_checkpoints/ — Jupyter autosave copies of + # notebooks, which go stale the moment the real notebook is regenerated. + # __pycache__ is DELIBERATELY left alone: it is an import-speed cache + # that costs seconds of every subsequent run to rebuild. + while IFS= read -r -d '' d; do + rel="${d#"$repo"/}" + n=$(git -C "$repo" clean -ndx -- "$rel" 2>/dev/null | wc -l) + [ "$n" -eq 0 ] && continue + show "remove $rel/" + [ "$DRY_RUN" = 1 ] || git -C "$repo" clean -qfdx -- "$rel" + done < <(find "$repo" -type d -name .ipynb_checkpoints -not -path '*/.git/*' -print0 2>/dev/null) fi # 4. Remove ignored, fully-untracked packaging products at managed library # roots. Keep assistant/workspace build products outside this narrow scope. # Never match nested domain directories named build, and never clean a # candidate containing tracked files even though git clean would retain them. - is_packaging_repo "$repo" || continue - while IFS= read -r -d '' d; do - rel="${d#"$repo"/}" - git -C "$repo" check-ignore -q -- "$rel" 2>/dev/null || continue - [ -z "$(git -C "$repo" ls-files -- "$rel" 2>/dev/null)" ] || continue - n=$(git -C "$repo" clean -ndx -- "$rel" 2>/dev/null | wc -l) - [ "$n" -eq 0 ] && continue - show "remove packaging directory $rel/" - [ "$DRY_RUN" = 1 ] || git -C "$repo" clean -qfdx -- "$rel" - done < <(find "$repo" -mindepth 1 -maxdepth 1 -type d \ - \( -name '*.egg-info' -o -name build \) -print0 2>/dev/null) + if is_packaging_repo "$repo"; then + while IFS= read -r -d '' d; do + rel="${d#"$repo"/}" + git -C "$repo" check-ignore -q -- "$rel" 2>/dev/null || continue + [ -z "$(git -C "$repo" ls-files -- "$rel" 2>/dev/null)" ] || continue + n=$(git -C "$repo" clean -ndx -- "$rel" 2>/dev/null | wc -l) + [ "$n" -eq 0 ] && continue + show "remove packaging directory $rel/" + [ "$DRY_RUN" = 1 ] || git -C "$repo" clean -qfdx -- "$rel" + done < <(find "$repo" -mindepth 1 -maxdepth 1 -type d \ + \( -name '*.egg-info' -o -name build \) -print0 2>/dev/null) + fi + + # 5. Compact the object store. `--auto` makes this a no-op until git's own + # loose-object threshold is crossed, so it is cheap to run every morning. + # Housekeeping must never break the sweep: warn and carry on. + if [ "$DRY_RUN" != 1 ]; then + git -C "$repo" gc --auto --quiet 2>/dev/null || warn "WARNING: git gc failed" + fi done echo diff --git a/bin/dataset_provenance.py b/bin/dataset_provenance.py new file mode 100755 index 0000000..6dcffcf --- /dev/null +++ b/bin/dataset_provenance.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""dataset_provenance.py — classify workspace datasets by WRITE SITE. + +`clean_slate.sh` must decide, for each *untracked* ``dataset//`` +directory, whether deleting it is free (a script regenerates it) or expensive +(it is real data that was downloaded, or nobody knows where it came from). + +Every workspace ignores ``dataset/`` wholesale and force-adds the real datasets +back, so "untracked" does NOT mean "regenerable". Nor is a name mention enough: +``scripts/interferometer/start_here.py`` names ``sdp81`` (real ALMA data it only +*reads*) three lines apart from the path of ``simulated_lens`` (which it +*writes*). A grep for the dataset name cannot tell those two apart — this helper +exists because that distinction has to be made on data flow, not text. + +__The rule: a dataset is regenerable only if a script demonstrably writes it__ + +For each candidate ``dataset//`` we scan every ``scripts/**/*.py`` +plus the repo's root-level ``*.py`` with ``ast`` and look for a *write site*: + + 1. **Path binding** — an assignment whose right-hand side is a path expression + resolving to the components ``dataset`` / ```` / ````. String + variables are resolved in source order, so ``dataset_type = "imaging"``, + ``dataset_name = "simple"``, ``dataset_path = Path("dataset", dataset_type, + dataset_name)`` resolves, as do ``Path("dataset") / t / n`` and + ``path.join("dataset", t, n)``. + 2. **Flow into a write call** — that bound variable (or a path derived from it, + ``uv_path = dataset_path / "uv.fits"``) appears in the arguments of a call + that writes: ``output_to_fits``, ``fits_imaging``, ``json.dump``, + ``open(..., "w")``, … (see ``WRITE_FUNCS``), or is passed to a helper + function in the same repo whose own body writes that parameter (one level of + interprocedural resolution — the ``simulators/util.py`` idiom). + +The analysis is **order sensitive**, which is the whole point: ``dataset_path`` +is rebound several times in one ``start_here.py``, and only the binding live at +the write call is credited. + +__Verdicts__ + +``DOWNLOADED`` the binding file also fetches over the network (urllib / + requests). Real data, cached not redistributed — deleting it + costs a large re-download. Wins over any write evidence. +``REGENERABLE`` positive write evidence, no network. Free to delete. +``ORPHAN`` no writer found. Kept, and reported so a human can look. + +Deletion requires positive evidence; every uncertainty resolves to ORPHAN. + +Usage: + dataset_provenance.py --repo dataset// ... +Prints one ``VERDICT `` line per candidate on stdout, and +``WARN unparseable `` lines on stderr for scripts that do not parse. +""" + +import argparse +import ast +import sys +import warnings +from pathlib import Path + +# Calls that put bytes on disk. Kept explicit rather than pattern-matched: every +# name added here makes the sweep delete more, so each one is a deliberate call. +# Deliberately absent: `aplt.fits_array`, which writes a single auxiliary array +# (a `mask_extra_galaxies.fits`) into a dataset folder. Real datasets get those +# dropped into them by their own `start_here.py` — writing one product is not +# evidence that the script generates the dataset. +WRITE_FUNCS = { + "output_to_fits", + "output_to_json", + "output_to_csv", + "fits_imaging", + "fits_interferometer", + "numpy_array_to_json", + "to_fits", + "savetxt", + "to_csv", + "dump", +} + +# Calls that pull bytes off the network. +NETWORK_FUNCS = {"urlretrieve", "urlopen"} +NETWORK_DOTTED = {"requests.get", "requests.urlretrieve", "requests.post"} +NETWORK_MODULES = {"urllib", "requests"} + +# Call names whose positional arguments are path components. +PATH_CALLS = {"Path", "PurePath", "PosixPath", "WindowsPath", "join"} + +# `open(path, mode)` counts as a write only for writing modes. +WRITE_MODES = ("w", "a", "x") + + +def _func_name(node): + """Trailing name of a call target: `pkg.mod.write` -> `write`, `f` -> `f`.""" + if isinstance(node, ast.Attribute): + return node.attr + if isinstance(node, ast.Name): + return node.id + return None + + +def _dotted_name(node): + """Full dotted call target where it is a plain attribute chain, else None.""" + parts = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if not isinstance(node, ast.Name): + return None + parts.append(node.id) + return ".".join(reversed(parts)) + + +def _const_str(node): + """The string value of a constant or all-constant f-string, else None.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr): + out = [] + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + out.append(value.value) + else: + return None + return "".join(out) + return None + + +def _is_path_shaped(node): + """True for `a / b` and `Path(...)` / `path.join(...)` expressions. + + Derived bindings are followed only through these forms. Without the guard, + `dataset = al.Imaging.from_fits(data_path=dataset_path / "data.fits")` would + make the *loaded dataset object* carry the real dataset's identity into the + next `fits_imaging(dataset=dataset, ...)` call, i.e. a read would be + laundered into a write. + """ + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + return True + return isinstance(node, ast.Call) and _func_name(node.func) in PATH_CALLS + + +def _is_write_call(node, name): + if name in WRITE_FUNCS: + return True + if name != "open": + return False + mode = None + if len(node.args) > 1: + mode = _const_str(node.args[1]) + for keyword in node.keywords: + if keyword.arg == "mode": + mode = _const_str(keyword.value) + return bool(mode) and mode[0] in WRITE_MODES + + +def _is_network_call(node, name): + if name in NETWORK_FUNCS: + return True + dotted = _dotted_name(node.func) + return dotted in NETWORK_DOTTED + + +class _Flow: + """Order-sensitive forward scan of one module (or one function body). + + `paths` maps a variable name to the set of opaque keys it currently refers + to. In the module pass a key is a `(type, name)` candidate; in the + helper-function pass it is a `("param", )` marker, so the same + machinery reports which parameters a helper writes. + """ + + def __init__(self, candidates, helpers=None, initial_paths=None): + self.candidates = candidates + self.helpers = helpers or {} + self.strs = {} + self.paths = dict(initial_paths or {}) + self.written = set() + self.network_keys = set() + self.bound = set() + self.uses_network = False + + # -- resolution ------------------------------------------------------- + + def _components(self, node): + """Path components of an expression; unresolvable pieces become None.""" + text = _const_str(node) + if text is not None: + return [part for part in text.split("/") if part] + if isinstance(node, ast.Name): + value = self.strs.get(node.id) + return [part for part in value.split("/") if part] if value else [None] + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + return self._components(node.left) + self._components(node.right) + if isinstance(node, ast.Call) and _func_name(node.func) in PATH_CALLS: + out = [] + for arg in node.args: + out.extend(self._components(arg)) + return out + return [None] + + def _candidate_of(self, node): + """The candidate a path expression names, else None.""" + if not (_is_path_shaped(node) or _const_str(node) is not None): + return None + components = self._components(node) + for i in range(len(components) - 2): + if components[i] != "dataset": + continue + pair = (components[i + 1], components[i + 2]) + if pair in self.candidates: + return pair + return None + + def _keys_of(self, node): + """Keys an expression refers to: a resolved path, or a bound variable.""" + keys = set() + direct = self._candidate_of(node) + if direct is not None: + keys.add(direct) + for sub in ast.walk(node): + if isinstance(sub, ast.Name) and sub.id in self.paths: + keys |= self.paths[sub.id] + elif sub is not node and isinstance(sub, ast.expr): + nested = self._candidate_of(sub) + if nested is not None: + keys.add(nested) + return keys + + def _call_keys(self, node): + keys = set() + for arg in node.args: + keys |= self._keys_of(arg) + for keyword in node.keywords: + keys |= self._keys_of(keyword.value) + return keys + + # -- statements ------------------------------------------------------- + + def run(self, body): + for node in body: + self._stmt(node) + + def _stmt(self, node): + if isinstance(node, (ast.Import, ast.ImportFrom)): + self._import(node) + return + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + # The body is analysed on its own in the helper pass; run it here + # too (module-level scripts define and call helpers inline), with + # the parameters shadowing any same-named module binding. + shadowed = self.paths, self.strs + self.paths = {k: v for k, v in self.paths.items() if k not in _params(node)} + self.strs = {k: v for k, v in self.strs.items() if k not in _params(node)} + self.run(node.body) + self.paths, self.strs = shadowed + return + if isinstance(node, ast.Assign): + self._exprs(node.value) + for target in node.targets: + self._bind(target, node.value) + return + if isinstance(node, ast.AnnAssign): + if node.value is not None: + self._exprs(node.value) + self._bind(node.target, node.value) + return + + for field, value in ast.iter_fields(node): + if field in ("body", "orelse", "finalbody", "handlers"): + continue + for expr in _expr_fields(value): + self._exprs(expr) + for field in ("body", "orelse", "finalbody"): + block = getattr(node, field, None) + if isinstance(block, list): + self.run(block) + for handler in getattr(node, "handlers", []): + self.run(handler.body) + + def _import(self, node): + names = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif node.module: + names = [node.module] + for name in names: + if name.split(".")[0] in NETWORK_MODULES: + self.uses_network = True + + def _bind(self, target, value): + if not isinstance(target, ast.Name): + return + name = target.id + text = _const_str(value) + if text is not None: + self.strs[name] = text + self.paths.pop(name, None) + return + self.strs.pop(name, None) + candidate = self._candidate_of(value) + if candidate is not None: + self.paths[name] = {candidate} + self.bound.add(candidate) + return + derived = set() + if _is_path_shaped(value): + for sub in ast.walk(value): + if isinstance(sub, ast.Name) and sub.id in self.paths: + derived |= self.paths[sub.id] + if derived: + self.paths[name] = derived + else: + self.paths.pop(name, None) + + def _exprs(self, node): + for sub in ast.walk(node): + if isinstance(sub, ast.Call): + self._call(sub) + + def _call(self, node): + name = _func_name(node.func) + if name is None: + return + if _is_network_call(node, name): + self.uses_network = True + self.network_keys |= self._call_keys(node) + if _is_write_call(node, name): + self.written |= self._call_keys(node) + return + params = self.helpers.get(name) + if not params: + return + order, writers = params + for index, arg in enumerate(node.args): + if index < len(order) and order[index] in writers: + self.written |= self._keys_of(arg) + for keyword in node.keywords: + if keyword.arg in writers: + self.written |= self._keys_of(keyword.value) + + +def _params(node): + args = node.args + names = [a.arg for a in list(args.posonlyargs) + list(args.args) + list(args.kwonlyargs)] + if args.vararg: + names.append(args.vararg.arg) + if args.kwarg: + names.append(args.kwarg.arg) + return names + + +def _expr_fields(value): + """Expressions reachable from a statement field, ignoring nested bodies.""" + if isinstance(value, ast.expr): + yield value + elif isinstance(value, list): + for item in value: + if isinstance(item, ast.expr): + yield item + elif isinstance(item, ast.withitem): + yield item.context_expr + if item.optional_vars is not None: + yield item.optional_vars + elif isinstance(value, ast.withitem): + yield value.context_expr + + +def _python_files(repo): + """`scripts/**/*.py` plus root-level `*.py`, skipping hidden dirs/caches.""" + files = sorted(repo.glob("*.py")) + scripts = repo / "scripts" + if scripts.is_dir(): + for path in sorted(scripts.rglob("*.py")): + parts = path.relative_to(repo).parts + if any(part.startswith(".") or part == "__pycache__" for part in parts): + continue + files.append(path) + return files + + +def _parse(path): + try: + with warnings.catch_warnings(): + # Workspace docstrings are full of `\d`-style sequences; their + # SyntaxWarnings are not our business and would drown real output. + warnings.simplefilter("ignore", SyntaxWarning) + return ast.parse(path.read_text(encoding="utf-8", errors="replace")) + except SyntaxError: + # A single broken script must not take the whole sweep down; say so and + # move on, which costs at worst an ORPHAN verdict (the safe direction). + print(f"WARN unparseable {path}", file=sys.stderr) + return None + except OSError as error: + print(f"WARN unreadable {path} ({error})", file=sys.stderr) + return None + + +def _helper_writers(trees): + """Map helper name -> (parameter order, parameters written by its body). + + Resolved to a fixpoint so `a()` calling `b()` calling `output_to_fits()` + still credits `a`'s parameter. Names are merged repo-wide; a collision only + ever means "some function of this name writes this parameter", which is + still positive evidence of a write. + """ + defs = {} + for tree in trees: + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + defs.setdefault(node.name, (node, _params(node))) + + helpers = {name: (order, set()) for name, (_, order) in defs.items()} + for _ in range(3): + changed = False + for name, (node, order) in defs.items(): + markers = {param: {("param", param)} for param in order} + flow = _Flow(set(), helpers=helpers, initial_paths=markers) + flow.run(node.body) + writers = {key[1] for key in flow.written if key[0] == "param"} + if not writers <= helpers[name][1]: + helpers[name] = (order, helpers[name][1] | writers) + changed = True + if not changed: + break + return helpers + + +def classify(repo, candidates): + """Return {(type, name): verdict} for the requested candidates.""" + pairs = set(candidates) + trees = [tree for tree in (_parse(p) for p in _python_files(repo)) if tree is not None] + helpers = _helper_writers(trees) + + regenerable, downloaded = set(), set() + for tree in trees: + flow = _Flow(pairs, helpers=helpers) + flow.run(tree.body) + regenerable |= flow.written + downloaded |= flow.network_keys + if flow.uses_network: + # A file that binds a dataset path and talks to the network is + # caching real data there (smacs0723 writes CSVs *derived from* its + # downloads). Blanket-marking every path it binds keeps them all. + downloaded |= flow.bound + + verdicts = {} + for pair in pairs: + if pair in downloaded: + verdicts[pair] = "DOWNLOADED" + elif pair in regenerable: + verdicts[pair] = "REGENERABLE" + else: + verdicts[pair] = "ORPHAN" + return verdicts + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo", required=True, help="repository root") + parser.add_argument("candidates", nargs="+", help="dataset// paths") + args = parser.parse_args(argv) + + repo = Path(args.repo) + if not repo.is_dir(): + parser.error(f"repo not found: {repo}") + + wanted = [] + for rel in args.candidates: + parts = Path(rel.rstrip("/")).parts + wanted.append((rel, (parts[1], parts[2]) if len(parts) >= 3 else None)) + + verdicts = classify(repo, [pair for _, pair in wanted if pair is not None]) + for rel, pair in wanted: + print(f"{verdicts.get(pair, 'ORPHAN')} {rel}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/wake_up/wake_up.md b/skills/wake_up/wake_up.md index 4d812e4..d2e8613 100644 --- a/skills/wake_up/wake_up.md +++ b/skills/wake_up/wake_up.md @@ -15,8 +15,11 @@ already produce. Keep it a thin conductor. ## Guardrail: auto only the safe steps -Auto-run **only the non-destructive steps** (sync, clean-slate — both git-aware -and reversible). Anything that deletes, edits, or bumps (stray cleanup, branch +Auto-run **only the recoverable steps** (sync, clean-slate — both git-aware). +Clean-slate does delete: untracked **regenerable** datasets, which the +workspace's own scripts write back on demand, plus generated cruft. It never +touches a tracked file, and datasets with no proven writer are reported, not +removed. Anything else that deletes, edits, or bumps (stray cleanup, branch deletion, version bumps) is **surfaced in the digest for the human to approve**, never done automatically. @@ -40,7 +43,10 @@ Run in order, then emit the digest. branch, ff-only; repos with real uncommitted work are skipped untouched. Note any left **off-main / dirty / behind / diverged**. 2. **Clean slate** — `bash PyAutoBrain/bin/clean_slate.sh` (`DRY_RUN=1` to - preview). Restore shipped datasets, clear `output/`/`scratch/` cruft. + preview). Restore shipped datasets, delete untracked regenerable ones + (recreated on demand by the scripts that write them), clear + `output/`/`scratch/` cruft. Note any **orphan dataset** lines — kept, not + deleted, and waiting on a human call. ### Everywhere (gh-API — mobile/codex-safe) 3. **Overnight sweep** — `bash PyAutoBrain/bin/overnight_status.sh`: latest diff --git a/tests/test_clean_slate.py b/tests/test_clean_slate.py index 42d2ce5..0e85c19 100644 --- a/tests/test_clean_slate.py +++ b/tests/test_clean_slate.py @@ -7,6 +7,10 @@ BRAIN_HOME = Path(__file__).resolve().parents[1] CLEAN_SLATE = BRAIN_HOME / "bin" / "clean_slate.sh" +PROVENANCE = BRAIN_HOME / "bin" / "dataset_provenance.py" + +# clean_slate.sh only sweeps datasets in the repos it knows are workspaces. +DATASET_REPO = "autolens_workspace" def _init_repo(root, name, ignore="*.egg-info/\nbuild/\n"): @@ -33,6 +37,93 @@ def _write(directory, name="generated.txt"): (directory / name).write_text("generated") +def _script(repo, rel, source): + """Drop a workspace script the provenance scan will parse.""" + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + return path + + +def _dataset(repo, dataset_type, name, filename="data.fits"): + """Create an untracked dataset directory holding one file.""" + path = repo / "dataset" / dataset_type / name + path.mkdir(parents=True, exist_ok=True) + if filename: + (path / filename).write_text("bytes") + return path + + +def _track(repo, *rel_paths): + subprocess.run(["git", "-C", str(repo), "add", "-f", *rel_paths], check=True) + + +def _provenance(repo, *candidates): + return subprocess.run( + ["python3", str(PROVENANCE), "--repo", str(repo), *candidates], + capture_output=True, + text=True, + ) + + +# A simulator that binds its dataset path from `dataset_type`/`dataset_name` and +# writes it — the shape every workspace `simulator.py` uses. +SIMULATOR = ''' +from pathlib import Path +import autolens as al +import autolens.plot as aplt + +dataset_type = "imaging" +dataset_name = "simple" +dataset_path = Path("dataset", dataset_type, dataset_name) + +aplt.fits_imaging( + dataset=dataset, + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", + overwrite=True, +) +''' + +# The #167 shape: a start_here.py that READS a real dataset by name and, further +# down the same file, REBINDS `dataset_path` and WRITES a simulated one. Only +# the simulated one may be deleted. +START_HERE = ''' +from pathlib import Path +import autolens as al +import autolens.plot as aplt + +dataset_name = "sdp81" +dataset_path = Path("dataset") / "interferometer" / dataset_name + +dataset = al.Interferometer.from_fits( + data_path=dataset_path / "data.fits", + noise_map_path=dataset_path / "noise_map.fits", +) + +al.output_to_fits(values=image.native, file_path=Path("image.fits"), overwrite=True) + +dataset_path = Path("dataset") / "imaging" / "simulated_lens" + +al.output_to_fits( + values=dataset.data.native, + file_path=dataset_path / "data.fits", + overwrite=True, +) +''' + +DOWNLOADER = ''' +from pathlib import Path +import urllib.request + +dataset_path = Path("dataset") / "cluster" / "smacs0723" +catalogue_path = dataset_path / "galcat.cat" + +if not catalogue_path.exists(): + urllib.request.urlretrieve("https://example.invalid/galcat.cat", catalogue_path) +''' + + def test_dry_run_reports_packaging_without_removing_it(tmp_path): repo = _init_repo(tmp_path, "PyAutoGalaxy") egg_info = repo / "autogalaxy.egg-info" @@ -82,3 +173,192 @@ def test_cleanup_is_root_scoped_ignored_and_tracked_safe(tmp_path): assert (protected / "build" / "tracked.txt").exists() assert (unignored / "local.egg-info").exists() assert (assistant / "build").exists() + + +def test_dataset_written_by_a_simulator_is_removed(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/simulator.py", SIMULATOR) + dataset = _dataset(repo, "imaging", "simple") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert not dataset.exists() + assert "remove 1 simulated dataset(s)" in result.stdout + + +def test_dataset_written_by_a_non_simulator_script_is_removed(tmp_path): + """The #167 gap: start_here.py writes datasets too, and no simulator does.""" + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/start_here.py", START_HERE) + dataset = _dataset(repo, "imaging", "simulated_lens") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert not dataset.exists() + + +def test_dataset_only_read_by_name_is_kept_while_written_one_is_removed(tmp_path): + """Name mention must never classify — only a write site may. + + One file names the real `sdp81` dataset (which it loads) and writes + `simulated_lens`. The reader must survive; the written one must not. + """ + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/start_here.py", START_HERE) + real = _dataset(repo, "interferometer", "sdp81") + simulated = _dataset(repo, "imaging", "simulated_lens") + + verdicts = _provenance( + repo, "dataset/interferometer/sdp81", "dataset/imaging/simulated_lens" + ) + assert verdicts.returncode == 0, verdicts.stderr + assert "ORPHAN dataset/interferometer/sdp81" in verdicts.stdout + assert "REGENERABLE dataset/imaging/simulated_lens" in verdicts.stdout + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert real.exists(), "a dataset that is only READ must never be deleted" + assert not simulated.exists() + assert "orphan dataset (no writer): dataset/interferometer/sdp81" in result.stdout + + +def test_downloaded_dataset_is_kept_silently(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/cluster/lenstool/data.py", DOWNLOADER) + dataset = _dataset(repo, "cluster", "smacs0723", filename="galcat.cat") + + verdicts = _provenance(repo, "dataset/cluster/smacs0723") + assert "DOWNLOADED dataset/cluster/smacs0723" in verdicts.stdout + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert dataset.exists() + assert "smacs0723" not in result.stdout, "downloaded data is kept without comment" + + +def test_tracked_dataset_is_never_deleted_and_is_restored(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/simulator.py", SIMULATOR) + dataset = _dataset(repo, "imaging", "simple") + _track(repo, "dataset/imaging/simple/data.fits") + (dataset / "data.fits").write_text("clobbered by a run") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert dataset.exists(), "a dataset holding tracked files is never a candidate" + assert (dataset / "data.fits").read_text() == "bytes" + assert "restore 1 modified dataset file(s)" in result.stdout + + +def test_dataset_with_no_writer_is_reported_as_an_orphan(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/simulator.py", SIMULATOR) + orphan = _dataset(repo, "imaging", "tutorial") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert orphan.exists() + assert "orphan dataset (no writer): dataset/imaging/tutorial" in result.stdout + + +def test_oversized_committed_dataset_warns_once_per_directory(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + dataset = _dataset(repo, "imaging", "cosmos_web_ring", filename=None) + for name in ("data.fits", "noise_map.fits"): + with open(dataset / name, "wb") as f: + f.truncate(6 * 1024 * 1024) + _track(repo, "dataset/imaging/cosmos_web_ring") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + warnings = [ + line for line in result.stdout.splitlines() if "committed dataset" in line + ] + assert len(warnings) == 1, warnings + assert "dataset/imaging/cosmos_web_ring is 12 MB (>5 MB)" in warnings[0] + + +def test_empty_dataset_directories_are_pruned(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + empty = repo / "dataset" / "imaging" / "leftover" + empty.mkdir(parents=True) + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert not empty.exists() + + +def test_ipynb_checkpoints_go_but_pycache_stays(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + checkpoints = repo / "scripts" / "imaging" / ".ipynb_checkpoints" + pycache = repo / "scripts" / "imaging" / "__pycache__" + _write(checkpoints, "start_here-checkpoint.ipynb") + _write(pycache, "util.cpython-311.pyc") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert not checkpoints.exists() + assert pycache.exists(), "__pycache__ is an import-speed cache, not cruft" + + +def test_dry_run_reports_dataset_actions_without_taking_them(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/start_here.py", START_HERE) + simulated = _dataset(repo, "imaging", "simulated_lens") + real = _dataset(repo, "interferometer", "sdp81") + empty = repo / "dataset" / "imaging" / "leftover" + empty.mkdir(parents=True) + checkpoints = repo / "scripts" / ".ipynb_checkpoints" + _write(checkpoints, "start_here-checkpoint.ipynb") + + result = _run(tmp_path, dry_run=True, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert "[dry-run] remove 1 simulated dataset(s)" in result.stdout + assert "[dry-run] remove 1 empty dataset directory" in result.stdout + assert "[dry-run] remove scripts/.ipynb_checkpoints/" in result.stdout + assert simulated.exists() and real.exists() + assert empty.exists() and checkpoints.exists() + + +def test_provenance_helper_failure_aborts_the_sweep(tmp_path): + """No fallback: a broken helper must stop the run, never guess.""" + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _dataset(repo, "imaging", "simple") + broken = tmp_path / "bin" + broken.mkdir() + (broken / "dataset_provenance.py").write_text("import sys\nsys.exit(3)\n") + (broken / "clean_slate.sh").write_text(CLEAN_SLATE.read_text()) + + env = {**os.environ, "PYAUTO_ROOT": str(tmp_path)} + result = subprocess.run( + ["bash", str(broken / "clean_slate.sh")], + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 1 + assert "dataset_provenance.py failed" in result.stderr + + +def test_unparseable_script_warns_without_crashing_the_sweep(tmp_path): + repo = _init_repo(tmp_path, DATASET_REPO, ignore="") + _script(repo, "scripts/imaging/simulator.py", SIMULATOR) + _script(repo, "scripts/imaging/broken.py", "def f(:\n") + dataset = _dataset(repo, "imaging", "simple") + + result = _run(tmp_path, packaging_only=False) + + assert result.returncode == 0, result.stderr + assert "WARN unparseable" in result.stderr + assert not dataset.exists()