diff --git a/agents/conductors/hygiene/_hygiene_extras.py b/agents/conductors/hygiene/_hygiene_extras.py index 6162e53..d9f6f5e 100644 --- a/agents/conductors/hygiene/_hygiene_extras.py +++ b/agents/conductors/hygiene/_hygiene_extras.py @@ -31,6 +31,16 @@ extra, and collect every third-party distribution reached. Expected coverage is the union of every library's ``[optional]`` closure — the set ``mode=release`` guarantees. The difference is the drift. + +*Which* libraries those are is DERIVED, never hard-coded: ``mode=release`` names +them (it installs each one's ``[optional]`` extra), so the same workflow defines +both sides of the comparison, and a library added to that leg is picked up here +instead of silently falling out of the scan. Each distribution it names is +resolved to a checkout by the ``project.name`` in that checkout's +``pyproject.toml`` — folder names carry no meaning. This also keeps organ code +free of tenant instance facts (the ``PyAutoMind/scripts/repos_sync.py`` tenant +firewall), the same way ``PyAutoHeart/heart/checks/release_run.py`` derives its +own release channel. """ from __future__ import annotations @@ -42,18 +52,16 @@ import tomllib from pathlib import Path -# The source libraries whose extras the workspace matrices install. Mirrors -# hygiene.sh's LIB_REPOS. -LIB_REPOS = ("PyAutoNerves", "PyAutoArray", "PyAutoFit", "PyAutoGalaxy", "PyAutoLens") - # The extra mode=release installs for every library, and therefore the coverage # mode=smoke is expected to match. RELEASE_EXTRA = "optional" WORKFLOW = Path("PyAutoHeart/.github/workflows/workspace-validation.yml") -# The install step whose requirement roots define the smoke leg's coverage. +# The install step whose requirement roots define the smoke leg's coverage, and +# the one that names the libraries (and therefore the coverage to match). SMOKE_STEP = re.compile(r"^\s*-\s*name:.*\[mode=smoke\]", re.IGNORECASE) +RELEASE_STEP = re.compile(r"^\s*-\s*name:.*\[mode=release\]", re.IGNORECASE) NEXT_STEP = re.compile(r"^\s*-\s*name:") PIP_INSTALL = re.compile(r"\bpip\s+install\b(?P.*)$") # A requirement's distribution name and its optional extras: "autoarray[optional]", @@ -76,25 +84,35 @@ def parse_requirement(token: str) -> tuple[str, tuple[str, ...]] | None: return canonical(match.group("name")), extras -def libraries(root: Path) -> dict[str, dict]: - """Map canonical distribution name -> its parsed pyproject, for each checkout.""" - found: dict[str, dict] = {} - for repo in LIB_REPOS: - pyproject = root / repo / "pyproject.toml" - if not pyproject.exists(): - continue +def libraries(root: Path) -> dict[str, tuple[str, dict]]: + """Map canonical distribution name -> (checkout dir, parsed pyproject). + + The library set is whatever `mode=release` installs — see the module + docstring — so a checkout counts iff the distribution it DECLARES + (`project.name`) is one that step names. Directory names are never matched + against, which is what keeps the list derived rather than hard-coded. + """ + declared = {name for name, _ in install_roots(root, RELEASE_STEP)} + if not declared: + return {} + + found: dict[str, tuple[str, dict]] = {} + for checkout in sorted(p for p in root.iterdir() if p.is_dir()): + pyproject = checkout / "pyproject.toml" try: data = tomllib.loads(pyproject.read_text()) - except (tomllib.TOMLDecodeError, UnicodeDecodeError): - continue # a malformed pyproject is the packaging mode's problem + except (OSError, tomllib.TOMLDecodeError, UnicodeDecodeError): + continue # absent, unreadable, or malformed — the packaging mode's problem name = data.get("project", {}).get("name") - if name: - found[canonical(name)] = data + if name and canonical(name) in declared: + found[canonical(name)] = (checkout.name, data) return found -def smoke_roots(root: Path) -> list[tuple[str, tuple[str, ...]]]: - """Requirement roots the smoke install step passes to pip, in order.""" +def install_roots( + root: Path, step: re.Pattern[str] +) -> list[tuple[str, tuple[str, ...]]]: + """Requirement roots the given install step passes to pip, in order.""" workflow = root / WORKFLOW if not workflow.exists(): return [] @@ -103,7 +121,7 @@ def smoke_roots(root: Path) -> list[tuple[str, tuple[str, ...]]]: block: list[str] = [] inside = False for line in lines: - if SMOKE_STEP.match(line): + if step.match(line): inside = True continue if inside and NEXT_STEP.match(line): @@ -188,29 +206,26 @@ def closure( def missing(root: Path) -> tuple[list[dict], str | None]: """Return (findings, skip-reason). A skip-reason means nothing was scannable.""" - libs = libraries(root) - if not libs: - return [], "no library checkouts under the scan root" + if not (root / WORKFLOW).exists(): + return [], f"{WORKFLOW} is not present under the scan root" - roots = smoke_roots(root) + checkouts = libraries(root) + if not checkouts: + return [], ( + f"no library checkout matches the [mode=release] install set in {WORKFLOW}" + ) + # The closure only needs each library's metadata; the checkout dir is + # carried alongside so a finding can name the repo to fix. + libs = {name: data for name, (_, data) in checkouts.items()} + + roots = install_roots(root, SMOKE_STEP) if not roots: return [], f"no smoke install step found in {WORKFLOW}" reached = closure(roots, libs) findings: list[dict] = [] - for repo in LIB_REPOS: - pyproject = root / repo / "pyproject.toml" - if not pyproject.exists(): - continue - # Resolve by the declared project name rather than the folder name. - try: - name = canonical(tomllib.loads(pyproject.read_text())["project"]["name"]) - except (tomllib.TOMLDecodeError, UnicodeDecodeError, KeyError): - continue - data = libs.get(name) - if data is None: - continue + for name, (repo, data) in sorted(checkouts.items()): if RELEASE_EXTRA not in { canonical(key) for key in (data.get("project", {}).get("optional-dependencies", {}) or {}) diff --git a/tests/test_hygiene_conductor.py b/tests/test_hygiene_conductor.py index 206833f..059320c 100644 --- a/tests/test_hygiene_conductor.py +++ b/tests/test_hygiene_conductor.py @@ -513,44 +513,50 @@ def test_optdeps_findings_reach_the_default_worklist(tmp_path): - name: "Install third-party deps (libs run from source) [mode=smoke]" if: needs.find_scripts.outputs.mode == 'smoke' run: | - pip install "autolens[optional]" + pip install "top-layer[optional]" {extra_installs}\ - name: "Install TestPyPI wheels [mode=release]" if: needs.find_scripts.outputs.mode == 'release' run: | pip install \\ - "autoarray[optional]==$V" \\ - "autolens[optional]==$V" + "base-layer[optional]==$V" \\ + "mid-layer[optional]==$V" \\ + "top-layer[optional]==$V" """ +# Keyed by CHECKOUT DIRECTORY, deliberately named nothing like the distribution +# each one declares: the scan derives its library set from the [mode=release] +# install step above and resolves each name through `project.name`, so a folder +# name must not be able to influence the result. (Instance-free by the same +# rule the tenant firewall applies to organ code.) _PYPROJECTS = { - # autonerves is the base layer; its [jax] extra is what the chain reaches. - "PyAutoNerves": """\ + # the base layer; its [jax] extra is what the chain reaches. + "checkout_a": """\ [project] -name = "autonerves" +name = "base-layer" dependencies = [] [project.optional-dependencies] jax = ["jax>=0.7"] -optional = ["autonerves[jax]", "astropy>=5.0"] +optional = ["base-layer[jax]", "astropy>=5.0"] """, - # autoarray declares an optional dep NO sibling's chain reaches — the drift. - "PyAutoArray": """\ + # mid-layer declares an optional dep NO sibling's chain reaches — the drift. + "checkout_b": """\ [project] -name = "autoarray" -dependencies = ["autonerves"] +name = "mid-layer" +dependencies = ["base-layer"] [project.optional-dependencies] -jax = ["autonerves[jax]"] -optional = ["autoarray[jax]", "numba", "tfp-nightly==0.26.0.dev1"] +jax = ["base-layer[jax]"] +optional = ["mid-layer[jax]", "numba", "tfp-nightly==0.26.0.dev1"] """, - # autolens[optional] chains to autolens[jax] -> autonerves[jax]; it never - # reaches autoarray[optional], which is the whole point of the scan. - "PyAutoLens": """\ + # top-layer[optional] chains to top-layer[jax] -> base-layer[jax]; it never + # reaches mid-layer[optional], which is the whole point of the scan. + "checkout_c": """\ [project] -name = "autolens" -dependencies = ["autoarray", "autonerves"] +name = "top-layer" +dependencies = ["mid-layer", "base-layer"] [project.optional-dependencies] -jax = ["autonerves[jax]"] -optional = ["autolens[jax]", "numba", "astropy>=5.0"] +jax = ["base-layer[jax]"] +optional = ["top-layer[jax]", "numba", "astropy>=5.0"] """, } @@ -558,10 +564,10 @@ def test_optdeps_findings_reach_the_default_worklist(tmp_path): def _write_extras_fixture(root, extra_installs=""): """Library checkouts + a PyAutoHeart workflow whose smoke leg under-installs. - `numba` is reachable from `autolens[optional]`. `astropy` is declared - optional by TWO libraries — autonerves (not reached) and autolens (reached) + `numba` is reachable from `top-layer[optional]`. `astropy` is declared + optional by TWO libraries — base-layer (not reached) and top-layer (reached) — so it must NOT be flagged; only a dependency no reached extra supplies is - drift. `tfp-nightly` is declared ONLY by `autoarray[optional]`, which the + drift. `tfp-nightly` is declared ONLY by `mid-layer[optional]`, which the chain never reaches -> FLAG. """ for repo, body in _PYPROJECTS.items(): @@ -588,13 +594,34 @@ def test_extras_flags_an_optional_dep_the_smoke_leg_never_installs(tmp_path): assert row["delegate"] == "/bug" finding = row["findings"][0] assert finding["dependency"] == "tfp-nightly" - assert finding["declared_by"] == ["autoarray[optional]"] + assert finding["declared_by"] == ["mid-layer[optional]"] + # Named by the CHECKOUT the distribution was resolved to, not by any + # assumption about what that folder is called. + assert finding["repos"] == ["checkout_b"] + + +def test_extras_resolves_libraries_by_distribution_not_by_folder(tmp_path): + # A checkout the [mode=release] step never installs is NOT a library, so its + # unreachable optional dep is none of this scan's business — even though it + # sits beside the real ones and declares the same shape of extra. + _write_extras_fixture(tmp_path) + outsider = tmp_path / "checkout_z" + outsider.mkdir() + (outsider / "pyproject.toml").write_text( + '[project]\nname = "not-a-library"\ndependencies = []\n' + '[project.optional-dependencies]\noptional = ["never-installed-pkg"]\n' + ) + + result = _run(["extras", "--json"], tmp_path) + + row = json.loads(result.stdout)["row"] + assert [f["dependency"] for f in row["findings"]] == ["tfp-nightly"] def test_extras_is_clean_once_the_declaring_extra_is_installed(tmp_path): # The house fix: install the declaring library's whole [optional] extra. _write_extras_fixture( - tmp_path, extra_installs=' pip install "autoarray[optional]"\n' + tmp_path, extra_installs=' pip install "mid-layer[optional]"\n' ) result = _run(["extras", "--json"], tmp_path)