diff --git a/.gitattributes b/.gitattributes index 57eb8a8807..f507ce96ca 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,9 @@ *.py linguist-language=python *.ipynb linguist-documentation .git_archival.txt export-subst + +# just refuses to format a CRLF file, so `just just-check` would fail on a Windows +# checkout with core.autocrlf=true before the contributor has changed anything. +Justfile text eol=lf +*/justfile text eol=lf +packages/*/justfile text eol=lf diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml index f0d7ab1471..960451cc4c 100644 --- a/.github/workflows/check_changelogs.yml +++ b/.github/workflows/check_changelogs.yml @@ -24,14 +24,20 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + - name: Check zarr-python changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py + run: just check-changelogs - name: Check zarr-metadata changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-metadata/changes + run: just check-changelogs packages/zarr-metadata/changes - name: Check zarr-indexing changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-indexing/changes + run: just check-changelogs packages/zarr-indexing/changes - name: Check zarr-http-server changelog entries - run: uv run --no-sync python ci/check_changelog_entries.py packages/zarr-http-server/changes + run: just check-changelogs packages/zarr-http-server/changes diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1441950c3f..a58ce40a9d 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -31,10 +31,19 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + # No interpreter is set up in this job on purpose. Benchmarks are only comparable + # across runs if the interpreter underneath them does not move: adding a pinned + # Python here measured a uniform ~13% slowdown on every benchmark, because hatch + # then built the environment on that interpreter instead of the runner's own. + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Run the benchmarks uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 env: ZARR_BENCHMARK_CLEAR_CACHE: '1' with: mode: walltime - run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed + run: HATCH_ENV=test.py3.12-minimal just benchmark-codspeed diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5e93efe230..990866a109 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,18 +23,22 @@ jobs: with: persist-credentials: false - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - - run: uv sync --group docs + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 # Fast source-level guards that need no built site, so they run before the (slower) # build for a quick failure: every public export is in the API reference, and no # docstring/Markdown carries reStructuredText markup that MkDocs won't render. - - run: uv run python ci/check_documented_exports.py docs/api - - run: uv run python ci/lint_docs.py + - run: just check-doc-exports + - run: just lint-docs # --strict turns warnings into errors, so a docs code block that fails to execute # at build time (e.g. a non-exec python fence disrupting a later exec="true" block) # fails CI instead of merging as a silent warning. - - run: uv run mkdocs build --strict + - run: just docs-build env: DISABLE_MKDOCS_2_WARNING: "true" NO_MKDOCS_2_WARNING: "true" - - run: uv run python ci/check_unlinked_types.py + - run: just check-doc-links continue-on-error: true diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 4c4d49a65c..a8e989f580 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -67,17 +67,24 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + # Two names for the same environment on purpose: `just setup` builds whatever + # HATCH_ENV points at, while `just gpu` reads GPU_HATCH_ENV so that an + # ambient HATCH_ENV can never redirect `pytest -m gpu` into a CPU environment. - name: Set Up Hatch Env env: HATCH_ENV: gputest.py${{ matrix.python-version }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup - name: Run Tests env: - HATCH_ENV: gputest.py${{ matrix.python-version }} + GPU_HATCH_ENV: gputest.py${{ matrix.python-version }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + just gpu - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index ff5382b4d4..3be6f3ba60 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -65,12 +65,16 @@ jobs: uses: pypa/hatch@f647ed70d49adb885f53a27d1c7f5bdaeacf2c60 with: version: '1.16.5' + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - name: Restore cached hypothesis directory id: restore-hypothesis-cache @@ -89,7 +93,7 @@ jobs: PYTEST_ADDOPTS: "--report-log=output-${{ matrix.python-version }}-log.jsonl" run: | echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE" - hatch env run --env "$HATCH_ENV" run-hypothesis + just hypothesis # explicitly save the cache so it gets updated, also do this even if it fails. - name: Save cached hypothesis directory diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 05354dd503..f47fbcac11 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,4 +30,26 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: enable-cache: true - - uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + # `uvx prek` builds each hook's environment from scratch, so cache them the + # way the prek action used to. Keyed on the hook config: a new pinned rev or + # a new hook is exactly when the cached environments stop being valid. + - name: Cache prek hook environments + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/prek + # Justfile is hashed too: prek_version lives there, and a bumped prek + # must not restore a store built by the previous one. No restore-keys + # fallback for the same reason. + key: prek-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml', 'Justfile') }} + - name: Lint + run: just lint + - name: Check justfile formatting + # After the linters, and never masking them: this is cosmetic, and a + # mis-formatted recipe should not cost someone their ruff/mypy results. + if: always() + run: just just-check diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb029bd153..0d7d396fff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,18 +69,22 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup - name: Run Tests env: HYPOTHESIS_PROFILE: ci HATCH_ENV: test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + just coverage - name: Upload coverage if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }} uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 @@ -118,17 +122,21 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env env: HATCH_ENV: ${{ matrix.dependency-set }} run: | - hatch env create "$HATCH_ENV" - hatch env run -e "$HATCH_ENV" list-env + just setup - name: Run Tests env: HATCH_ENV: ${{ matrix.dependency-set }} run: | - hatch env run --env "$HATCH_ENV" run-coverage + just coverage - name: Upload coverage uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: @@ -153,12 +161,17 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Set Up Hatch Env run: | - hatch run doctest:pip list + HATCH_ENV=doctest just setup - name: Run Tests run: | - hatch run doctest:test + just doctest benchmarks: name: Benchmark smoke test @@ -177,11 +190,16 @@ jobs: uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install Hatch run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 - name: Run Benchmarks env: ZARR_BENCHMARK_CLEAR_CACHE: '1' run: | - hatch env run --env "test.py3.13-minimal" run-benchmark + HATCH_ENV=test.py3.13-minimal just benchmark test-complete: name: Test complete diff --git a/.github/workflows/zarr-http-server.yml b/.github/workflows/zarr-http-server.yml index b90528d40f..4a8fcca101 100644 --- a/.github/workflows/zarr-http-server.yml +++ b/.github/workflows/zarr-http-server.yml @@ -53,11 +53,10 @@ jobs: - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Sync test dependency groups # The examples group carries the deps the README examples need, so the # test that reads a served array back with a zarr client runs here @@ -80,11 +79,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff run: just lint @@ -106,11 +104,10 @@ jobs: - name: Set up Python run: uv python install 3.12 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Run mypy @@ -132,11 +129,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.github/workflows/zarr-indexing.yml b/.github/workflows/zarr-indexing.yml index 5d74c6a5e9..9af8ed1e3d 100644 --- a/.github/workflows/zarr-indexing.yml +++ b/.github/workflows/zarr-indexing.yml @@ -40,11 +40,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} # The suite imports nothing from `zarr`; it runs against the repo-root @@ -73,11 +72,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff # The ruff version pin lives in packages/zarr-indexing/justfile. run: just lint @@ -102,11 +100,10 @@ jobs: - name: Sync test dependency group run: uv sync --group test --python 3.12 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run pyright # The pyright invocation lives in packages/zarr-indexing/justfile. run: just typecheck @@ -127,11 +124,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs # The strict mkdocs build lives in packages/zarr-indexing/justfile. run: just docs-check diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index 521a4f270c..3c42f4810f 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -43,11 +43,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Sync test dependency group @@ -69,11 +68,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run ruff run: just lint @@ -93,11 +91,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Run pyright # The pyright version and interpreter pins live in the justfile. run: just typecheck @@ -118,11 +115,10 @@ jobs: with: enable-cache: true - name: Install just - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 with: - # Pin the tool, not just the action: without this the action - # installs whatever just is newest at run time. - just-version: 1.58.0 + repo: casey/just + version: 1.58.0 - name: Build docs run: just docs-check diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dddf8449a4..872eb2be80 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,15 +19,24 @@ build: fi install: - pip install --upgrade pip - - pip install .[remote] --group docs + # The docs recipes resolve their toolchain from uv.lock, so uv is all that is + # needed here besides just itself. + - pip install uv==0.12.9 + # No GitHub Action is available on Read the Docs, so fetch the official just + # release and verify it against the checksum casey/just publishes. Installed + # into the build virtualenv's bin, which is already on PATH. + - | + curl -fsSL -o /tmp/just.tar.gz https://github.com/casey/just/releases/download/1.58.0/just-1.58.0-x86_64-unknown-linux-musl.tar.gz + echo "4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b84931d /tmp/just.tar.gz" | sha256sum -c - + tar -xzf /tmp/just.tar.gz -C "$READTHEDOCS_VIRTUALENV_PATH/bin" just pre_build: - | if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; then - towncrier build --version Unreleased --yes; + just changelog-build --version Unreleased --yes; fi build: html: - - mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html + - just docs-build --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: mkdocs.yml diff --git a/Justfile b/Justfile new file mode 100644 index 0000000000..6e56644207 --- /dev/null +++ b/Justfile @@ -0,0 +1,165 @@ +# Development and CI verbs live here; Hatch owns Python environments in pyproject.toml. +# Install: uv tool install hatch==1.16.5 && uv tool install rust-just==1.58.0 +# Select test dependencies/interpreter: HATCH_ENV=test.py3.13-minimal just test +# On Windows, use Git Bash (the same shell used by the test workflow). +set shell := ["bash", "-eu", "-o", "pipefail", "-c"] +set windows-shell := ["bash", "-eu", "-o", "pipefail", "-c"] +set positional-arguments + +hatch_env := env("HATCH_ENV", "test.py3.12-optional") +# Deliberately a different variable from HATCH_ENV: `just gpu` must not inherit a +# CPU test environment that happens to be exported in the caller's shell, which +# would run `pytest -m gpu` against an environment built without the gpu feature. +gpu_env := env("GPU_HATCH_ENV", "gputest.py3.12") +# Pinned so a prek release cannot change what CI lints without a commit here. +prek_version := "0.5.3" +# Documentation and changelog tooling resolves from uv.lock, not a hatch environment. +# There is only ever one docs toolchain, so it can be locked and hash-verified, and +# dependabot's uv ecosystem keeps it current. Hatch still owns the test environments, +# which exist per interpreter and per dependency set and cannot live in one lockfile. +docs_run := "uv run --frozen --group docs" +# The hatch docs environment used to set these; they belong with the mkdocs calls now. +mkdocs_env := "DISABLE_MKDOCS_2_WARNING=true NO_MKDOCS_2_WARNING=true" + +# List available recipes +default: + @just --list + +# List available Python environments +envs: + hatch env show + +# Create the selected Python environment and list its installed packages +setup: && list-env + hatch env create {{ quote(hatch_env) }} + +# List packages in the selected Python environment +list-env: + hatch run {{ quote(hatch_env) }}:pip list + +# Run unit tests; pass pytest arguments, e.g. just test -k 'array and resize' +test *args: + hatch run {{ quote(hatch_env) }}:pytest --ignore tests/benchmarks "$@" + +# Run unit tests and write coverage.xml and junit.xml +coverage *args: + hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy "$@" + hatch run {{ quote(hatch_env) }}:coverage xml + +# Run unit tests and generate an HTML coverage report +coverage-html *args: + hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest --ignore tests/benchmarks "$@" + hatch run {{ quote(hatch_env) }}:coverage html + +# Serve the HTML coverage report (default port 8000) +coverage-serve *args: + hatch run {{ quote(hatch_env) }}:python -m http.server -d htmlcov "$@" + +# Run slow Hypothesis tests and write coverage.xml +hypothesis *args: + hatch run {{ quote(hatch_env) }}:coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* "$@" + hatch run {{ quote(hatch_env) }}:coverage xml + +# Validate executable documentation code blocks +doctest *args: + hatch run doctest:pytest tests/test_docs.py -v "$@" + +# Run the benchmark suite +benchmark *args: + hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" + +# Run benchmarks under CodSpeed +benchmark-codspeed *args: + hatch run {{ quote(hatch_env) }}:pytest tests/benchmarks --codspeed "$@" + +# Run GPU tests with coverage; select the environment with GPU_HATCH_ENV +gpu *args: + HATCH_ENV={{ quote(gpu_env) }} just coverage -m gpu "$@" + +# Build documentation (warnings are errors) +docs-build *args: + {{ mkdocs_env }} {{ docs_run }} mkdocs build --strict "$@" + +# Serve documentation with live reload +docs-serve *args: + {{ mkdocs_env }} {{ docs_run }} mkdocs serve --watch src "$@" + +# Check that every public export has API documentation +check-doc-exports *args: + {{ docs_run }} python ci/check_documented_exports.py docs/api "$@" + +# Check documentation source conventions +lint-docs *args: + {{ docs_run }} python ci/lint_docs.py "$@" + +# Report unlinked types in built documentation +check-doc-links *args: + {{ docs_run }} python ci/check_unlinked_types.py "$@" + +# Run source documentation checks followed by a strict build +docs-check: check-doc-exports lint-docs docs-build + +# Run all pre-commit hooks (ruff, codespell, mypy, repo-review, ...) +lint *args: + uvx prek@{{ prek_version }} run --show-diff-on-failure --color=always --all-files "$@" + +# Run hooks with a custom selection, e.g. just hooks run --last-commit +hooks +args: + uvx prek@{{ prek_version }} "$@" + +# prek is installed as a persistent uv tool rather than run through uvx: the hook shim +# prek writes into .git/hooks hard-codes the binary path it was installed from and falls +# back to `prek` on PATH, and a uvx archive path stops existing at the next cache prune. +# Install local pre-commit hooks +hooks-install: + uv tool install prek=={{ prek_version }} + prek install + +# Type-check the library using the locked tooling environment +typecheck *args: + uv run --frozen mypy "$@" + +# Check that uv.lock is in sync with pyproject.toml +lock-check: + uv lock --check + +# Update the dependency lockfile +lock *args: + uv lock "$@" + +# Build the source distribution and wheel +build *args: + hatch build "$@" + +# Create a changelog fragment (interactive without arguments) +changelog *args: + {{ docs_run }} towncrier create "$@" + +# Preview the next release's changelog +changelog-draft *args: + {{ docs_run }} towncrier build --draft --version Unreleased "$@" + +# Build release notes; pass --version and --yes when preparing a release +changelog-build *args: + {{ docs_run }} towncrier build "$@" + +# Check changelog filenames (default: changes/; accepts a package changes directory) +check-changelogs *args: + uv run --no-project python ci/check_changelog_entries.py "$@" + +# Check recipe formatting of the root Justfile and every package justfile +just-check: + just --fmt --check + shopt -s nullglob; for f in packages/*/justfile; do just --justfile "$f" --fmt --check; done + +# Run a zarr-metadata recipe, or list its recipes with no arguments +zarr-metadata *args: + just --justfile packages/zarr-metadata/justfile "$@" + +# Run a zarr-indexing recipe, or list its recipes with no arguments +zarr-indexing *args: + just --justfile packages/zarr-indexing/justfile "$@" + +# Run a zarr-http-server recipe, or list its recipes with no arguments +zarr-http-server *args: + just --justfile packages/zarr-http-server/justfile "$@" diff --git a/changes/4372.misc.md b/changes/4372.misc.md new file mode 100644 index 0000000000..55a800a3f3 --- /dev/null +++ b/changes/4372.misc.md @@ -0,0 +1 @@ +Define development and CI commands in a root Justfile, with Hatch managing Python environments and documentation tooling resolving from `uv.lock`. The Hatch script tables are removed, so invocations like `hatch env run --env test.py3.12-optional run-coverage` become `just coverage`; see the contributing guide for the full set. Extracted from [#4096](https://github.com/zarr-developers/zarr-python/pull/4096). diff --git a/docs/contributing.md b/docs/contributing.md index 369e60110a..45e401e064 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -80,19 +80,52 @@ git remote add upstream git@github.com:zarr-developers/zarr-python.git ### Creating a development environment -To work with the Zarr source code, it is recommended to use [hatch](https://hatch.pypa.io/latest/index.html) to create and manage development environments. Hatch will automatically install all Zarr dependencies using the same versions as are used by the core developers and continuous integration services. Assuming you have a Python 3 interpreter already installed, and you have cloned the Zarr source code and your current working directory is the root of the repository, you can do something like the following: +The root `Justfile` defines development and CI commands. [just](https://just.systems/) +runs these commands, while [Hatch](https://hatch.pypa.io/latest/index.html) manages +the Python environments declared in `pyproject.toml`. Install +[uv](https://docs.astral.sh/uv/getting-started/installation/) first, then the two task +tools. `uv tool install` puts them in their own environments, which a plain `pip install` +cannot do on a Python that marks itself externally managed (Debian, Ubuntu, Homebrew): ```bash -pip install hatch -hatch env show # list all available environments +uv tool install hatch==1.16.5 +uv tool install rust-just==1.58.0 # or any option from https://just.systems/man/en/packages.html +just # list available commands +just envs # list Python environments +just setup # create the default test environment +just test ``` -To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.: +Test recipes default to `test.py3.12-optional`. Set `HATCH_ENV` to select a different +interpreter or dependency set, just as CI does. `just gpu` reads `GPU_HATCH_ENV` +instead, so an exported `HATCH_ENV` cannot silently send GPU tests to an +environment built without the `gpu` feature, and `just doctest` always runs in the +`doctest` environment. On Windows, run these commands in +Git Bash. ```bash -hatch env run --env test.py3.12-optional run +HATCH_ENV=test.py3.13-minimal just test +HATCH_ENV=min_deps just coverage +HATCH_ENV=upstream just coverage +GPU_HATCH_ENV=gputest.py3.12 just gpu +just test tests/test_array.py -k 'resize and not async' ``` +Arguments after the recipe name are forwarded to the underlying tool. Use +`just --show test` to inspect a command. Package-specific commands live in the +`justfile` inside each package directory. The root recipes delegate to these files, +so you can also run package commands from the repository root: + +```bash +just zarr-metadata # list this package's recipes +just zarr-metadata test +just zarr-indexing test-tensorstore +just zarr-http-server docs-check +``` + +The package justfile sets the working directory and defines the command and its +environment. Root recipes forward arguments without duplicating those definitions. + ### Creating a branch Before you do any new work or submit a pull request, please open an issue on GitHub to report the bug or propose the feature you'd like to add. @@ -125,10 +158,10 @@ Again, any conflicts need to be resolved before submitting a pull request. ### Running the test suite -Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: +Zarr includes a suite of unit tests. The simplest way to run the unit tests is to invoke: ```bash -hatch env run --env test.py3.12-optional run +just test ``` All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. @@ -137,46 +170,36 @@ All tests are automatically run via GitHub Actions for every pull request and mu All code must conform to the PEP8 standard. Regarding line length, lines up to 100 characters are allowed, although please try to keep under 90 wherever possible. -`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager that is fully compatible with `.pre-commit-config.yaml` files. `prek` can be installed locally by running: - -```bash -uv tool install prek -``` - -or: - -```bash -pip install prek -``` +`Zarr` uses a set of git hooks managed by [`prek`](https://github.com/j178/prek), a fast, Rust-based pre-commit hook manager compatible with `.pre-commit-config.yaml`. The recipes pin the prek version: `just lint` and `just hooks` run it through `uvx`, and `just hooks-install` installs it as a persistent `uv tool` so the git hook can find it on later commits. The hooks can be installed locally by running: ```bash -prek install +just hooks-install ``` This will run the checks every time a commit is created locally. The checks will by default only run on the files modified by a commit, but the checks can be triggered for all the files by running: ```bash -prek run --all-files +just lint ``` You can also run hooks only for files in a specific directory: ```bash -prek run --directory src/zarr +just hooks run --directory src/zarr ``` Or run hooks for files changed in the last commit: ```bash -prek run --last-commit +just hooks run --last-commit ``` To list all available hooks: ```bash -prek list +just hooks list ``` If you would like to skip the failing checks and push the code for further discussion, use the `--no-verify` option with `git commit`. @@ -188,7 +211,7 @@ If you would like to skip the failing checks and push the code for further discu Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: ```bash -hatch env run --env test.py3.12-optional run-coverage +just coverage ``` will automatically run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. @@ -196,7 +219,7 @@ will automatically run the test suite with coverage and produce an XML coverage You can also generate an HTML coverage report by running: ```bash -hatch env run --env test.py3.12-optional run-coverage-html +just coverage-html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. @@ -210,15 +233,17 @@ Zarr uses mkdocs for documentation, hosted on readthedocs.org. Documentation is The documentation can be built locally by running: ```bash -hatch --env docs run build +just docs-build ``` +`just docs-check` also runs the documentation source checks used in CI. + The resulting built documentation will be available in the `site` folder. -Hatch can also be used to serve continuously updating version of the documentation during development at [http://127.0.0.1:8000/](http://127.0.0.1:8000/). This can be done by running: +`just docs-serve` serves a continuously updating version of the documentation during development at [http://127.0.0.1:8000/](http://127.0.0.1:8000/). This can be done by running: ```bash -hatch --env docs run serve +just docs-serve ``` #### Adding executable code blocks in the documentation @@ -320,10 +345,10 @@ Sometimes, you may want the documentation to build quicker. You can disable code ### Changelog -zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `towncrier create` with your development environment, which will prompt you for the issue number, change type, and the news text: +zarr-python uses [towncrier](https://towncrier.readthedocs.io/en/stable/tutorial.html) to manage release notes. Most pull requests should include at least one news fragment describing the changes. To add a release note, you'll need the GitHub issue or pull request number and the type of your change (`feature`, `bugfix`, `doc`, `removal`, `misc`). With that, run `just changelog`, which will prompt you for the issue number, change type, and the news text: ```bash -towncrier create +just changelog ``` Alternatively, you can manually create the files in the `changes` directory using the naming convention `{issue-number}.{change-type}.md`. @@ -437,6 +462,18 @@ Features in `zarr.experimental` carry no stability guarantees. They may be chang Zarr uses [pytest-benchmark](https://pytest-benchmark.readthedocs.io/en/latest/) for running performance benchmarks as part of our test suite. The benchmarks are found in `tests/benchmarks`. By default pytest is configured to run these benchmarks as plain tests (i.e., no benchmarking). To run -a benchmark with timing measurements, use the `--benchmark-enable` when invoking `pytest`. +a benchmark with timing measurements, run `just benchmark`. Pass pytest arguments +to select benchmarks, for example `just benchmark -k test_morton_order`. The benchmarks are run as part of the continuous integration suite through [codspeed](https://app.codspeed.io/zarr-developers/zarr-python). + +## Building distributions and maintaining dependencies + +`just just-check` verifies that the root `Justfile` and each package `justfile` are +formatted the way CI expects; `just --fmt` rewrites them in place if it complains. + +Run `just build` to produce a source distribution and wheel in `dist/`. +Use `just lock-check` to check the dependency lockfile, or `just lock` to update it. +`just typecheck` runs the type checker independently of the other lint hooks. +Preview release notes with `just changelog-draft`; use `just check-changelogs` +to validate fragment names, optionally passing a package's `changes/` directory. diff --git a/lychee.toml b/lychee.toml index dccb3001dc..9bb9be5caf 100644 --- a/lychee.toml +++ b/lychee.toml @@ -16,7 +16,7 @@ exclude_path = [ # URL patterns to ignore (regex, matched against the full URL). exclude = [ - # Local docs preview server shown in the contributing guide ("hatch run serve"), + # Local docs preview server shown in the contributing guide ("just docs-serve"), # documentation of a command rather than a reachable link. '^https?://0\.0\.0\.0', '^https?://(localhost|127\.0\.0\.1)(:\d+)?', diff --git a/packages/zarr-http-server/justfile b/packages/zarr-http-server/justfile index d65b1b53b7..5daac00ebc 100644 --- a/packages/zarr-http-server/justfile +++ b/packages/zarr-http-server/justfile @@ -10,6 +10,9 @@ # newest release: when ruff 0.16 began selecting BLE001 under the root # config's `B` prefix, this job failed on rules the pinned ruff never enforced, # with no code change to blame. Bump alongside the pre-commit rev. +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + ruff_version := "0.16.0" # List available recipes @@ -21,7 +24,7 @@ default: # silently skipping. # Run the test suite; extra args are passed to pytest test *args: - uv run --group test --group examples pytest tests {{ args }} + uv run --group test --group examples pytest tests "$@" # Lint the package sources and tests lint: diff --git a/packages/zarr-indexing/justfile b/packages/zarr-indexing/justfile index e32f177ff0..29f6ce885f 100644 --- a/packages/zarr-indexing/justfile +++ b/packages/zarr-indexing/justfile @@ -2,6 +2,9 @@ # directory as the working directory regardless of where `just` is invoked. # List available recipes +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + default: @just --list @@ -10,7 +13,7 @@ default: # overlay, using the same test invocation as CI. # Run the test suite; extra args are passed to pytest test *args: - uv run --project ../.. --group test --with-editable . python -m pytest tests src/zarr_indexing {{ args }} + uv run --project ../.. --group test --with-editable . python -m pytest tests src/zarr_indexing "$@" # TensorStore is the oracle for the parity suites, which skip without it. It # ships binary wheels only, so it rides in as a run-time overlay rather than @@ -18,7 +21,7 @@ test *args: # gate the CI job that calls this on the matrix version. # Run the tensorstore parity suites; extra args are passed to pytest test-tensorstore *args: - uv run --project ../.. --group test --with-editable . --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py {{ args }} + uv run --project ../.. --group test --with-editable . --with 'tensorstore>=0.1.84' python -m pytest tests/test_ndsel_tensorstore.py tests/test_tensorstore_parity.py "$@" # Lint with the same invocation CI uses. Ruff is pinned to the repo-wide # version in the root .pre-commit-config.yaml; bump together. diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index 0f1861ed7d..843286b89f 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -2,12 +2,15 @@ # directory as the working directory regardless of where `just` is invoked. # List available recipes +# Quoted arguments must survive delegation from the root Justfile. +set positional-arguments + default: @just --list # Run the test suite; extra args are passed to pytest test *args: - uv run --group test pytest tests {{ args }} + uv run --group test pytest tests "$@" # Lint with the same invocation CI uses lint: diff --git a/pyproject.toml b/pyproject.toml index 4be9d3342e..a4dafe4d6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,8 @@ include = [ "/examples", "/mkdocs.yml", "/mkdocs_hooks.py", + "/Justfile", + "/ci", ] [project] @@ -205,25 +207,6 @@ matrix.deps.dependency-groups = [ {value = "remote-tests", if = ["optional"]}, ] -[tool.hatch.envs.test.scripts] -run-coverage = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks --junitxml=junit.xml -o junit_family=legacy {args:}", - "coverage xml", -] -run-coverage-html = [ - "coverage run --source=src -m pytest --ignore tests/benchmarks {args:}", - "coverage html", -] -run = "pytest --ignore tests/benchmarks" -run-verbose = "run-coverage --verbose" -run-hypothesis = [ - "coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* {args:}", - "coverage xml", -] -run-benchmark = "pytest --benchmark-enable tests/benchmarks" -serve-coverage-html = "python -m http.server -d htmlcov 8000" -list-env = "pip list" - [tool.hatch.envs.gputest] template = "test" extra-dependencies = [ @@ -238,13 +221,6 @@ features = ["gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.12", "3.13"] -[tool.hatch.envs.gputest.scripts] -run-coverage = [ - "coverage run --source=src -m pytest -m gpu --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks {args:}", - "coverage xml", -] -run = "pytest -m gpu --ignore tests/benchmarks" - [tool.hatch.envs.upstream] template = 'test' python = "3.14" @@ -289,20 +265,6 @@ extra-dependencies = [ [tool.hatch.envs.default] installer = "uv" -[tool.hatch.envs.docs] -features = ['remote'] -dependency-groups = ['docs'] - -[tool.hatch.envs.docs.env-vars] -DISABLE_MKDOCS_2_WARNING = "true" -NO_MKDOCS_2_WARNING = "true" - -[tool.hatch.envs.docs.scripts] -serve = "mkdocs serve --watch src" -build = "mkdocs build" -check = "mkdocs build --strict" -readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT/html" - [tool.hatch.envs.doctest] description = "Test environment for validating executable code blocks in documentation" features = ['remote'] @@ -311,10 +273,6 @@ extra-dependencies = [ "pytest-examples", ] -[tool.hatch.envs.doctest.scripts] -test = "pytest tests/test_docs.py -v" -list-env = "pip list" - [tool.ruff] line-length = 100 force-exclude = true diff --git a/tests/test_docs.py b/tests/test_docs.py index 874d90d4e5..6361591a5e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -210,7 +210,7 @@ def test_test_only_blocks_come_last() -> None: Because we cannot statically tell which later blocks are state-dependent, this guard enforces the simple, safe convention only for the blocks we author this way (test="true" marker-bound examples like s3/gpu). It is NOT a complete build-hazard - check -- the authoritative check is `mkdocs build --strict` (the docs:check CI job), + check -- the authoritative check is `mkdocs build --strict` (`just docs-build` in CI), which catches the exec="false" case too. This guard just turns the common test-only case into a fast, local failure.""" # Collect, per published-docs file, the start lines of test-only and exec blocks.