diff --git a/.github/workflows/zarr-metadata.yml b/.github/workflows/zarr-metadata.yml index df7d96cc1c..b5f56dd508 100644 --- a/.github/workflows/zarr-metadata.yml +++ b/.github/workflows/zarr-metadata.yml @@ -1,5 +1,8 @@ name: zarr-metadata +# Job steps delegate to packages/zarr-metadata/justfile, the single source of +# truth for this package's verbs; CI owns only the python matrix and caching. + on: push: branches: [main] @@ -39,12 +42,14 @@ jobs: uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true + - name: Install just + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 - name: Set up Python ${{ matrix.python-version }} run: uv python install ${{ matrix.python-version }} - name: Sync test dependency group run: uv sync --group test --python ${{ matrix.python-version }} - name: Run pytest - run: uv run --group test pytest tests + run: just test ruff: name: ruff @@ -59,8 +64,10 @@ jobs: persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - name: Install just + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 - name: Run ruff - run: uvx ruff check . + run: just lint pyright: name: pyright @@ -77,19 +84,35 @@ jobs: uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Sync test dependency group - run: uv sync --group test --python 3.11 + - name: Install just + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 - name: Run pyright - # Pinned to the last version that types PEP 661 sentinels in class - # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115). - # Unpin when the fix lands. - run: uv run --group test --with 'pyright==1.1.404' pyright src + # The pyright version and interpreter pins live in the justfile. + run: just typecheck + + docs: + name: docs + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: packages/zarr-metadata + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: true + - name: Install just + uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4 + - name: Build docs + run: just docs-check zarr-metadata-complete: name: zarr-metadata complete - needs: [test, ruff, pyright] + needs: [test, ruff, pyright, docs] if: always() runs-on: ubuntu-latest steps: diff --git a/mkdocs.yml b/mkdocs.yml index 46bfc1764c..87aaf23430 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -92,6 +92,7 @@ nav: - ' zarr.testing.utils': api/zarr/testing/utils.md - ' zarr.zeros': api/zarr/functions/zeros.md - ' zarr.zeros_like': api/zarr/functions/zeros_like.md + - 'zarr-metadata ↪': https://zarr-metadata.readthedocs.io/ - release-notes.md - contributing.md hooks: diff --git a/packages/zarr-metadata/.readthedocs.yaml b/packages/zarr-metadata/.readthedocs.yaml new file mode 100644 index 0000000000..b89846f570 --- /dev/null +++ b/packages/zarr-metadata/.readthedocs.yaml @@ -0,0 +1,20 @@ +# Read the Docs configuration for the zarr-metadata docs site, separate from +# the zarr-python site configured by the repo-root .readthedocs.yaml. The RTD +# project for zarr-metadata must set its configuration-file path to +# packages/zarr-metadata/.readthedocs.yaml. +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + jobs: + install: + - pip install --upgrade pip + - pip install ./packages/zarr-metadata --group packages/zarr-metadata/pyproject.toml:docs + build: + html: + - mkdocs build --strict -f packages/zarr-metadata/mkdocs.yml --site-dir $READTHEDOCS_OUTPUT/html + +mkdocs: + configuration: packages/zarr-metadata/mkdocs.yml diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 69b80d7332..6b6b172aec 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -2,6 +2,8 @@ Python types, models, and validators for Zarr v2 and v3 metadata. +Documentation: + ## What this is Two layers and an optional integration: @@ -82,6 +84,23 @@ store I/O. The models begin and end at the metadata documents themselves — `from_key_value` / `to_key_value` map documents to store keys and bytes, and everything past that belongs to consumer libraries. +## Developing + +Package-scoped development commands live in the [`justfile`](./justfile) +(requires [just](https://github.com/casey/just)): + +``` +just test # run the test suite (extra args go to pytest) +just lint # ruff, same invocation as CI +just typecheck # pyright, pinned to the version CI uses +just docs-check # strict build of the docs site +just check # all of the above +just docs-serve # serve the docs site locally +``` + +Run them from this directory, or from anywhere in the repository as +`just packages/zarr-metadata/`. + ## Releasing The package version is derived from git tags by `hatch-vcs`. Tags must diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md new file mode 100644 index 0000000000..2aa39ab161 --- /dev/null +++ b/packages/zarr-metadata/docs/api/index.md @@ -0,0 +1,31 @@ +--- +title: API reference +--- + +# API reference + +The package is organized to mirror the structure of the Zarr specifications: + +- [`zarr_metadata.model`](model.md) — frozen-dataclass document models, + structural validators, loc-aware parsers, and the `UNSET` sentinel +- [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types + over the models +- [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents + (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`) +- [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 + documents, with subpackages for [chunk grids](v3/chunk_grid.md), + [chunk key encodings](v3/chunk_key_encoding.md), [codecs](v3/codec.md), + and [data types](v3/data_type.md) + +Every public name is also re-exported at the top level, so +`from zarr_metadata import ZarrV3ArrayMetadataJSON` and +`from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON` are equivalent. + +## Common types + +A few cross-cutting aliases are exported only from the top-level +`zarr_metadata` namespace: + +::: zarr_metadata.JSONValue + +::: zarr_metadata.ZarrV3NamedConfigJSON diff --git a/packages/zarr-metadata/docs/api/model.md b/packages/zarr-metadata/docs/api/model.md new file mode 100644 index 0000000000..c82ba98f2d --- /dev/null +++ b/packages/zarr-metadata/docs/api/model.md @@ -0,0 +1,5 @@ +--- +title: model +--- + +::: zarr_metadata.model diff --git a/packages/zarr-metadata/docs/api/pydantic.md b/packages/zarr-metadata/docs/api/pydantic.md new file mode 100644 index 0000000000..edecb416a7 --- /dev/null +++ b/packages/zarr-metadata/docs/api/pydantic.md @@ -0,0 +1,5 @@ +--- +title: pydantic +--- + +::: zarr_metadata.pydantic diff --git a/packages/zarr-metadata/docs/api/v2.md b/packages/zarr-metadata/docs/api/v2.md new file mode 100644 index 0000000000..2fe5b6ec56 --- /dev/null +++ b/packages/zarr-metadata/docs/api/v2.md @@ -0,0 +1,17 @@ +--- +title: v2 +--- + +::: zarr_metadata.v2 + options: + members: false + +::: zarr_metadata.v2.array + +::: zarr_metadata.v2.group + +::: zarr_metadata.v2.attributes + +::: zarr_metadata.v2.codec + +::: zarr_metadata.v2.consolidated diff --git a/packages/zarr-metadata/docs/api/v3/chunk_grid.md b/packages/zarr-metadata/docs/api/v3/chunk_grid.md new file mode 100644 index 0000000000..724b1c9d8d --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/chunk_grid.md @@ -0,0 +1,11 @@ +--- +title: chunk_grid +--- + +::: zarr_metadata.v3.chunk_grid + options: + members: false + +::: zarr_metadata.v3.chunk_grid.regular + +::: zarr_metadata.v3.chunk_grid.rectilinear diff --git a/packages/zarr-metadata/docs/api/v3/chunk_key_encoding.md b/packages/zarr-metadata/docs/api/v3/chunk_key_encoding.md new file mode 100644 index 0000000000..bb063deb25 --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/chunk_key_encoding.md @@ -0,0 +1,11 @@ +--- +title: chunk_key_encoding +--- + +::: zarr_metadata.v3.chunk_key_encoding + options: + members: false + +::: zarr_metadata.v3.chunk_key_encoding.default + +::: zarr_metadata.v3.chunk_key_encoding.v2 diff --git a/packages/zarr-metadata/docs/api/v3/codec.md b/packages/zarr-metadata/docs/api/v3/codec.md new file mode 100644 index 0000000000..cb96d2c7d5 --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/codec.md @@ -0,0 +1,25 @@ +--- +title: codec +--- + +::: zarr_metadata.v3.codec + options: + members: false + +::: zarr_metadata.v3.codec.blosc + +::: zarr_metadata.v3.codec.bytes + +::: zarr_metadata.v3.codec.cast_value + +::: zarr_metadata.v3.codec.crc32c + +::: zarr_metadata.v3.codec.gzip + +::: zarr_metadata.v3.codec.scale_offset + +::: zarr_metadata.v3.codec.sharding_indexed + +::: zarr_metadata.v3.codec.transpose + +::: zarr_metadata.v3.codec.zstd diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md new file mode 100644 index 0000000000..f482c33201 --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -0,0 +1,45 @@ +--- +title: data_type +--- + +::: zarr_metadata.v3.data_type + options: + members: false + +::: zarr_metadata.v3.data_type.bool + +::: zarr_metadata.v3.data_type.int8 + +::: zarr_metadata.v3.data_type.int16 + +::: zarr_metadata.v3.data_type.int32 + +::: zarr_metadata.v3.data_type.int64 + +::: zarr_metadata.v3.data_type.uint8 + +::: zarr_metadata.v3.data_type.uint16 + +::: zarr_metadata.v3.data_type.uint32 + +::: zarr_metadata.v3.data_type.uint64 + +::: zarr_metadata.v3.data_type.float16 + +::: zarr_metadata.v3.data_type.float32 + +::: zarr_metadata.v3.data_type.float64 + +::: zarr_metadata.v3.data_type.complex64 + +::: zarr_metadata.v3.data_type.complex128 + +::: zarr_metadata.v3.data_type.raw + +::: zarr_metadata.v3.data_type.bytes + +::: zarr_metadata.v3.data_type.string + +::: zarr_metadata.v3.data_type.numpy_datetime64 + +::: zarr_metadata.v3.data_type.numpy_timedelta64 diff --git a/packages/zarr-metadata/docs/api/v3/index.md b/packages/zarr-metadata/docs/api/v3/index.md new file mode 100644 index 0000000000..f20267d372 --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/index.md @@ -0,0 +1,15 @@ +--- +title: v3 +--- + +::: zarr_metadata.v3 + options: + members: false + +::: zarr_metadata.v3.ZarrV3MetadataFieldJSON + +::: zarr_metadata.v3.array + +::: zarr_metadata.v3.group + +::: zarr_metadata.v3.consolidated diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md new file mode 100644 index 0000000000..2004f2dc54 --- /dev/null +++ b/packages/zarr-metadata/docs/index.md @@ -0,0 +1,102 @@ +--- +title: zarr-metadata +--- + +# zarr-metadata + +Basic tools for modelling Zarr metadata, with minimal dependencies. + +`zarr-metadata` is developed in the +[zarr-python repository](https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata) +and released independently of `zarr` itself. Install it with: + +``` +pip install zarr-metadata +``` + +## Who needs this + +This library might be useful to you if your software interacts with Zarr metadata documents. + +## What this is + +This library is *not* a full Zarr implementation. Instead, it's a collection of data structures and routines that +closely model the content of the Zarr specifications, such as: + +- **Typed JSON shapes** ([`zarr_metadata.v2`](api/v2.md) and + [`zarr_metadata.v3`](api/v3/index.md)): `TypedDict` definitions and + `Literal` aliases for the JSON documents specified by the + [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) and + [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) + specifications, plus types for + [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/) and a + few widely-used-but-unspecified entities (e.g. consolidated metadata). +- **Document models** ([`zarr_metadata.model`](api/model.md)): canonical + frozen-dataclass models of whole metadata documents, with structural + validators, loc-aware parsers, and store-key (de)serialization. A document + produced by `to_json` shares no mutable state with the model that produced + it. +- **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), + requires Pydantic 2.13 or newer): each model as a Pydantic field type that + validates raw documents through the same strict parser. + +## What this is for + +The public `TypedDict` definitions describe the static JSON shape of Zarr +metadata. For strict, loc-aware validation of JSON loaded from disk, use the +model parser: + +```python +import json +from zarr_metadata.model import ZarrV3ArrayMetadata + +with open("zarr.json", "rb") as f: + raw = json.load(f) + +metadata = ZarrV3ArrayMetadata.from_json(raw) +``` + +The optional Pydantic integration delegates raw input to the same strict +parser and returns the same normalized model class: + +```python +from pydantic import TypeAdapter +import zarr_metadata.pydantic as zmp + +metadata = TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(raw) +encoded = metadata.to_key_value()["zarr.json"] +``` + +A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape +adapter, not a Zarr conformance validator; it may coerce values or discard +members that the strict model parser rejects. + +## Validation boundary + +The model validators enforce the declared document structure and a small set +of context-free consistency rules, including fixed format literals, finite +JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one +`dimension_names` entry per array dimension. They do not interpret extension +names or configurations, resolve codec pipelines, or decide whether a data +type, chunk grid, codec, or storage transformer is supported. Those decisions +belong to consumer implementations. + +## Scope + +At minimum, this library supports what Zarr-Python needs: the complete +Zarr v2 and v3 specs, consolidated metadata, and a subset of the metadata +defined in `zarr-extensions`. We are generally open to contributions that +add types, models, or structural validation for Zarr metadata with a +published spec. + +Runtime array behavior is out of scope: nothing here encodes or decodes +chunks, resolves codec or data type names to implementations, or performs +store I/O. The models begin and end at the metadata documents themselves — +`from_key_value` / `to_key_value` map documents to store keys and bytes, +and everything past that belongs to consumer libraries. + +## Reference + +- [API reference](api/index.md) +- [Changelog](https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md) +- [License (MIT)](https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/LICENSE.txt) diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile new file mode 100644 index 0000000000..0f1861ed7d --- /dev/null +++ b/packages/zarr-metadata/justfile @@ -0,0 +1,58 @@ +# Development verbs for the zarr-metadata package. Recipes run with this +# directory as the working directory regardless of where `just` is invoked. + +# List available recipes +default: + @just --list + +# Run the test suite; extra args are passed to pytest +test *args: + uv run --group test pytest tests {{ args }} + +# Lint with the same invocation CI uses +lint: + uvx ruff check . + +# Pinned to the last pyright that types PEP 661 sentinels in class attributes +# correctly; 1.1.405+ regressed (microsoft/pyright#11115). Unpin when fixed. +pyright_version := "1.1.404" + +# CI runs pyright on python 3.11; the pinned pyright predates 3.14, whose +# stdlib it cannot parse, so pin the interpreter to match CI. +# Type-check the package sources +typecheck: + uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright src + +# Run everything CI runs for this package +check: lint typecheck test docs-check + +# Preview the changelog that the next release would generate +changelog-draft: + uvx towncrier build --draft --version Unreleased + +# Build this package's documentation site, warnings as errors +docs-check: + env DISABLE_MKDOCS_2_WARNING=true uv run --group docs mkdocs build --strict + +# With no argument, uses port 8000 if free, otherwise an ephemeral free port; +# an explicitly requested port is used as-is so a conflict fails loudly. +# Serve this package's documentation site +docs-serve port="": + #!/usr/bin/env bash + set -euo pipefail + port="{{ port }}" + if [ -z "$port" ]; then + port=$(uv run --group docs python -c ' + import socket + s = socket.socket() + try: + s.bind(("127.0.0.1", 8000)) + except OSError: + s.close() + s = socket.socket() + s.bind(("127.0.0.1", 0)) + print(s.getsockname()[1]) + s.close() + ') + fi + exec env DISABLE_MKDOCS_2_WARNING=true uv run --group docs mkdocs serve -a "localhost:$port" diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml new file mode 100644 index 0000000000..40912d6251 --- /dev/null +++ b/packages/zarr-metadata/mkdocs.yml @@ -0,0 +1,103 @@ +site_name: zarr-metadata +repo_name: zarr-developers/zarr-python +repo_url: https://github.com/zarr-developers/zarr-python +edit_uri: edit/main/packages/zarr-metadata/docs/ +site_description: Spec-defined metadata types, models, and validators for Zarr v2 and v3. +site_author: Davis Bennett +site_url: !ENV [READTHEDOCS_CANONICAL_URL, 'https://zarr-metadata.readthedocs.io/'] +docs_dir: docs +use_directory_urls: true + +nav: + - index.md + - API Reference: + - api/index.md + - ' zarr_metadata.model': api/model.md + - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.v2': api/v2.md + - ' zarr_metadata.v3': + - api/v3/index.md + - ' zarr_metadata.v3.chunk_grid': api/v3/chunk_grid.md + - ' zarr_metadata.v3.chunk_key_encoding': api/v3/chunk_key_encoding.md + - ' zarr_metadata.v3.codec': api/v3/codec.md + - ' zarr_metadata.v3.data_type': api/v3/data_type.md + - Changelog: https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md + +watch: + - src + +theme: + language: en + name: material + + palette: + # Light mode + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to light mode + + font: + text: Roboto + code: Roboto Mono + + features: + - content.code.annotate + - content.code.copy + - navigation.indexes + - navigation.instant + - navigation.tracking + - search.suggest + - search.share + +plugins: + - autorefs + - search + - mkdocstrings: + enable_inventory: true + handlers: + python: + paths: [src] + options: + allow_inspection: true + docstring_section_style: list + docstring_style: numpy + inherited_members: true + line_length: 60 + separate_signature: true + show_root_heading: true + show_signature_annotations: true + show_source: true + show_symbol_type_toc: true + signature_crossrefs: true + show_if_no_docstring: true + extensions: + - griffe_inherited_docstrings + + inventories: + - https://docs.python.org/3/objects.inv + - https://zarr.readthedocs.io/en/stable/objects.inv + +markdown_extensions: + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - pymdownx.details + - pymdownx.superfences + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index edc4b696a6..6e97d26409 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -44,10 +44,21 @@ Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" Issues = "https://github.com/zarr-developers/zarr-python/issues" Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/CHANGELOG.md" -Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md" +Documentation = "https://zarr-metadata.readthedocs.io/" [dependency-groups] test = ["pytest", "pydantic>=2.13", "jsonschema"] +docs = [ + # Pins match the zarr-python docs environment in the repo-root + # pyproject.toml so the two sites render with the same toolchain. + "mkdocs-material==9.7.6", + "mkdocs==1.6.1", + "mkdocstrings==1.0.4", + "mkdocstrings-python==2.0.5", + "griffe-inherited-docstrings==1.1.3", + # mkdocstrings uses ruff to format rendered signatures + "ruff==0.15.20", +] [tool.hatch.version] source = "vcs"