From d07ae202f71f4e4c14faf28a3de5b8c10978aca1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 07:41:38 +0000 Subject: [PATCH 1/3] ci: gate PyAutoHands's own test suite on pull_request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyAutoHands had zero PR checks. None of its three workflows is triggered by pull_request: python_matrix.yml is a weekly cron over the five libraries' suites, navigator_check.yml is workflow_call-only, and release.yml's pytest step runs inside the library matrix path — PyAutoHands is checked out beside it only as a helper, so tests/ was never collected. ~300 tests covering build_util, env_config, result_collector, check_navigator, clone_seed and the release-notes tooling ran in no CI at all. Adds tests.yml on the shape of PyAutoBrain's tests.yml and PyAutoHeart's heart-tests.yml: push-to-main + pull_request, 3.12/3.13, pytest only, with cancel-in-progress restricted to non-main refs (a cancelled main run reads as red to Heart, which counts `cancelled` in FAILURE_CONCLUSIONS). Dependencies are named explicitly rather than -r requirements.txt, which would pull jupyterlab + ipykernel: pytest, PyYAML, ipynb-py-convert (a CLI binary build_util shells out to), Pillow (generate_markdown's PNG path). Derived empirically from --collect-only upward, not assumed. Also repairs test_python_matrix_workflow.py, which the missing gate had let drift. b038fdc deliberately promoted Python 3.14 to a required leg of the unit and smoke matrices and retired the soft experimental_python_314 job, updating the summary job and its banner to match — but not this test, which still asserted the pre-promotion contract and had been failing unnoticed since. The workflow is correct and internally consistent; the guard test is what was stale, so it is updated to the promoted shape rather than the workflow being reverted. It now also asserts 3.14 cannot regrow a continue-on-error home where its failures stop counting. Baseline: 301 passed, 3 skipped in ~6s on both 3.12 and 3.13. The 3 skips are notebook-execution tests that self-skip on absent nbformat/jupyter. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qRQyavh7MT89E6UtX5aEm --- .github/workflows/tests.yml | 68 ++++++++++++++++++++++++++++ tests/test_python_matrix_workflow.py | 33 +++++++------- 2 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..425a42b --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,68 @@ +name: Hands Tests + +# PyAutoHands's own unit suite — the executor's self-test. Until this existed, +# PyAutoHands PRs carried ZERO check runs: `python_matrix.yml` is a weekly cron +# over the five *libraries'* suites, `navigator_check.yml` is `workflow_call` +# only (invoked by the workspaces), and `release.yml`'s pytest step runs inside +# `${{ matrix.project.path }}` — a matrix of the five libraries, with +# PyAutoHands checked out beside them only as a helper. So the ~300 tests +# covering build_util (script/notebook execution, per-script timeouts, +# clean-skip exits), env_config (profile discovery, per-script env, JAX +# marking, workspace precedence), result_collector, check_navigator, +# clone_seed, the release/Slack notes and the workflow parsers ran in no CI at +# all, and a Hands PR's only gate was whatever the authoring session ran +# locally. +# +# That gap had already bitten: b038fdc promoted Python 3.14 to a required +# matrix leg and retired the experimental_python_314 job, but left +# test_python_matrix_workflow.py asserting the pre-promotion contract. The +# guard test sat failing with nothing to report it. +# +# Deliberately ONLY pytest. It must not invoke `autohands generate` / `run_all` +# / `pre_build` against live workspaces, and must not reach the network — those +# need sibling workspace checkouts and belong to release.yml and the scheduled +# drivers, not to a PR gate. What runs here is stdlib plus four small packages, +# so it stays fast (~6s) and flake-free. +# +# Dependencies are named explicitly rather than `-r requirements.txt`, which +# would drag in jupyterlab + ipykernel and turn a 6s gate into a slow one: +# pytest the runner +# PyYAML env_config / validate_env_profiles / the workflow parsers +# ipynb-py-convert a CLI binary build_util shells out to (not an import) +# Pillow generate_markdown's PNG optimisation path +# Three tests self-skip on absent nbformat/jupyter; that is intentional — those +# belong to the notebook-execution path the release workflow covers. + +# One run per commit: PR events carry the CI; pushes only build main. +# Superseded runs are cancelled on PR refs only — a cancelled main run would +# read as red CI (cancelled is in Heart's FAILURE_CONCLUSIONS). +on: + push: + branches: [main] + pull_request: + +concurrency: + group: hands-tests-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install (the whole dependency set — see header) + run: pip install pytest PyYAML ipynb-py-convert Pillow + - name: Run tests + run: pytest tests/ -q diff --git a/tests/test_python_matrix_workflow.py b/tests/test_python_matrix_workflow.py index 8b02e47..86491b4 100644 --- a/tests/test_python_matrix_workflow.py +++ b/tests/test_python_matrix_workflow.py @@ -19,31 +19,30 @@ def test_required_matrices_cover_only_supported_python_versions(): assert jobs["unit_tests"]["strategy"]["matrix"]["python-version"] == [ "3.12", "3.13", + "3.14", ] assert jobs["smoke_tests"]["strategy"]["matrix"]["python-version"] == [ "3.12", "3.13", + "3.14", ] -def test_python_314_is_isolated_and_non_required(): +def test_python_314_is_a_required_leg_not_an_isolated_experiment(): + """3.14 was promoted to a required leg of both matrices (b038fdc, following + the PyAutoFit#1439 forkserver fix), which retired the soft + `experimental_python_314` job. Guard the promoted shape: 3.14 must sit in + the required matrices above, and must not quietly regrow a + `continue-on-error` home where its failures stop counting.""" jobs = load_workflow()["jobs"] - experimental = jobs["experimental_python_314"] - - assert experimental["continue-on-error"] is True - assert experimental["strategy"]["matrix"]["python-version"] == ["3.14"] - assert len(experimental["strategy"]["matrix"]["project"]) == 5 - assert "experimental_python_314" in jobs["summary"]["needs"] - assert "continue-on-error" not in jobs["unit_tests"] - assert "continue-on-error" not in jobs["smoke_tests"] - - record_step = next( - step for step in experimental["steps"] - if step.get("name") == "Record experimental cell result" - ) - assert record_step["if"] == "always()" - assert "job.status" in record_step["run"] - assert "does not cover workspace scripts" in record_step["run"] + + assert "experimental_python_314" not in jobs + + for name in ("unit_tests", "smoke_tests"): + assert "3.14" in jobs[name]["strategy"]["matrix"]["python-version"] + assert "continue-on-error" not in jobs[name] + + assert sorted(jobs["summary"]["needs"]) == ["smoke_tests", "unit_tests"] def test_no_below_floor_success_or_banner_contract_remains(): From 65be53713d1d59b783b510f8f5187ae7c645347c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 07:46:50 +0000 Subject: [PATCH 2/3] docs: correct the tests.yml header on what the gate does not cover CI reported 300 passed / 4 skipped against a local 301 / 3. The extra skip is test_workspace_config_precedence.test_actual_workspace_files_exist, which walks repo_root.parent / asserting each of the six workspaces owns its config/build yaml files, and skips at the first one absent. In a bare PyAutoHands checkout it asserts nothing; in a full local workspace it asserts all six. That is a real hole in the gate, and it is accepted rather than closed: fixing it means six extra checkouts for a repo-layout invariant, coupling a 6s gate to six other repos. But the header claimed only three notebook-related skips, so it hid the one skip that actually costs coverage. Both baselines are now stated. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qRQyavh7MT89E6UtX5aEm --- .github/workflows/tests.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 425a42b..d85d238 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,8 +30,20 @@ name: Hands Tests # PyYAML env_config / validate_env_profiles / the workflow parsers # ipynb-py-convert a CLI binary build_util shells out to (not an import) # Pillow generate_markdown's PNG optimisation path -# Three tests self-skip on absent nbformat/jupyter; that is intentional — those -# belong to the notebook-execution path the release workflow covers. +# FOUR tests self-skip here, and they are two different kinds: +# - 3 on absent nbformat/jupyter (test_run_notebook_cwd, +# test_notebook_skip_exit) — intentional, that is the notebook-execution +# path release.yml covers. +# - 1 because this checkout has no sibling workspaces: +# test_workspace_config_precedence.test_actual_workspace_files_exist walks +# `repo_root.parent / ` asserting each of the six workspaces owns +# its config/build/{no_run,profile_smoke,visualise_notebooks}.yaml, and +# skips at the first one absent. In a full local workspace it asserts all +# six; here it asserts nothing. +# That second one is a real hole in this gate, accepted deliberately: closing it +# means six extra checkouts for a repo-layout invariant, which would couple a 6s +# gate to six other repos. It is a local/developer check, not a PR check. Local +# baseline is therefore 301 passed / 3 skipped; CI is 300 passed / 4 skipped. # One run per commit: PR events carry the CI; pushes only build main. # Superseded runs are cancelled on PR refs only — a cancelled main run would From 38341634a409bf36fea3ad6234868a9e05376b5e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 07:56:45 +0000 Subject: [PATCH 3/3] ci: run the Hands gate on 3.14 and tie it to the supported set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit python_matrix.yml declares 3.12/3.13/3.14 required (b038fdc), but the new self-test gate only ran 3.12/3.13 — so Hands's own code was unverified on a version the repo calls supported. Verified the suite on 3.14 first (302 passed, 3 skipped, ~6s — identical to 3.12/3.13), then added it to the matrix. Also adds test_self_test_gate_tracks_the_supported_python_set, asserting tests.yml's matrix equals python_matrix.yml's required unit_tests matrix. Drift between a version policy and the file guarding it is exactly how this module went stale in the first place; tying the lists together means promoting or dropping a version has to touch both. Checked the assertion is not vacuous — both sides parse to a non-empty list. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013qRQyavh7MT89E6UtX5aEm --- .github/workflows/tests.yml | 5 ++++- tests/test_python_matrix_workflow.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d85d238..ffffcbc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,10 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.13"] + # Tracks the required set python_matrix.yml declares supported (3.14 + # promoted in b038fdc). test_python_matrix_workflow.py asserts the two + # stay equal, so this list cannot silently fall behind that policy. + python-version: ["3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/tests/test_python_matrix_workflow.py b/tests/test_python_matrix_workflow.py index 86491b4..9025f7a 100644 --- a/tests/test_python_matrix_workflow.py +++ b/tests/test_python_matrix_workflow.py @@ -6,6 +6,9 @@ WORKFLOW = ( Path(__file__).resolve().parents[1] / ".github" / "workflows" / "python_matrix.yml" ) +SELF_TEST_WORKFLOW = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "tests.yml" +) AUTOHANDS = Path(__file__).resolve().parents[1] / "bin" / "autohands" @@ -13,6 +16,25 @@ def load_workflow(): return yaml.safe_load(WORKFLOW.read_text()) +def test_self_test_gate_tracks_the_supported_python_set(): + """Hands's own gate (tests.yml) must run the same Python set python_matrix.yml + declares required. + + Drift between a version policy and the file guarding it is exactly how this + module went stale: b038fdc promoted 3.14 in python_matrix.yml and nothing + reported that the guard still asserted the old shape. Tying the two lists + together means promoting or dropping a version has to touch both. + """ + required = load_workflow()["jobs"]["unit_tests"]["strategy"]["matrix"][ + "python-version" + ] + gate = yaml.safe_load(SELF_TEST_WORKFLOW.read_text())["jobs"]["pytest"][ + "strategy" + ]["matrix"]["python-version"] + + assert gate == required + + def test_required_matrices_cover_only_supported_python_versions(): jobs = load_workflow()["jobs"]