From 252fc552383ab4f8b72fdee0e2ee71e0371ce5bd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 3 Sep 2026 20:58:51 +0200 Subject: [PATCH 001/107] feat(zarr-metadata): composition rules layer, shape-exact entity validators, create_* factories Adds `zarr_metadata.rules`: cross-field judgments over whole documents (fill value vs data type, codec pipeline ordering, chunk-grid geometry, sharding and transpose against the array each codec actually receives, struct field rules, dimension-name counts), registered where they are defined and dispatched per entity; `validate_*` / `is_*` / `parse_*` / `check_*` front doors for readers; `v3._shape` validators derived from the entity TypedDicts; and one `create_*` factory per document TypedDict. The two rank checks (v3 `dimension_names` vs `shape`, v2 `chunks` vs `shape`) move from the structural validator into the rules layer. The pydantic field types now run the rules layer before normalizing, so they are strictly stronger than before rather than weaker. Relative to d-v-b/zarr-python#296 this drops the incremental builder, the extension-point provenance table, `TypeIs` codec guards, fill-value propagation through codec chains (no rule read it), registry introspection helpers, and a duplicated v2 consolidated envelope check; renames the `.zarray`/`.zgroup` factories to `zarray`/`zgroup`; and requires an endianness on the `bytes` codec inside `index_codecs`. Split from d-v-b/zarr-python#296 (part 2 of 3). Assisted-by: ClaudeCode:claude-fable-5-1 --- packages/zarr-metadata/README.md | 37 +- .../zarr-metadata/changes/296.feature.3.md | 14 + .../zarr-metadata/changes/296.feature.5.md | 17 + .../zarr-metadata/changes/296.feature.6.md | 13 + packages/zarr-metadata/changes/296.feature.md | 51 ++ packages/zarr-metadata/docs/api/builder.md | 5 + packages/zarr-metadata/docs/api/index.md | 6 + packages/zarr-metadata/docs/api/rules.md | 5 + packages/zarr-metadata/docs/index.md | 18 +- packages/zarr-metadata/mkdocs.yml | 2 + packages/zarr-metadata/pyproject.toml | 2 +- .../src/zarr_metadata/builder/__init__.py | 32 + .../src/zarr_metadata/builder/_create.py | 333 +++++++++ .../src/zarr_metadata/model/_validation.py | 51 +- .../src/zarr_metadata/pydantic.py | 63 +- .../src/zarr_metadata/rules/__init__.py | 73 ++ .../src/zarr_metadata/rules/_documents.py | 187 +++++ .../src/zarr_metadata/rules/_engine.py | 115 +++ .../zarr_metadata/rules/_entities/__init__.py | 27 + .../rules/_entities/bytes_codec.py | 116 +++ .../rules/_entities/cast_value.py | 32 + .../src/zarr_metadata/rules/_entities/gzip.py | 31 + .../rules/_entities/numpy_time.py | 38 + .../rules/_entities/rectilinear_grid.py | 125 ++++ .../rules/_entities/regular_grid.py | 60 ++ .../zarr_metadata/rules/_entities/sharding.py | 150 ++++ .../rules/_entities/struct_dtype.py | 164 +++++ .../rules/_entities/transpose.py | 78 ++ .../src/zarr_metadata/rules/_pipeline.py | 119 ++++ .../src/zarr_metadata/rules/_registry.py | 319 +++++++++ .../src/zarr_metadata/rules/_result.py | 101 +++ .../src/zarr_metadata/rules/_spec.py | 156 ++++ .../src/zarr_metadata/rules/_v2_array.py | 54 ++ .../src/zarr_metadata/rules/_v3_array.py | 428 +++++++++++ .../src/zarr_metadata/rules/_v3_group.py | 85 +++ .../src/zarr_metadata/v3/_extension_points.py | 54 ++ .../src/zarr_metadata/v3/_shape.py | 666 ++++++++++++++++++ .../src/zarr_metadata/v3/codec/__init__.py | 15 + .../src/zarr_metadata/v3/codec/kind.py | 66 ++ .../src/zarr_metadata/v3/data_type/raw.py | 12 +- .../zarr-metadata/tests/builder/__init__.py | 0 .../tests/builder/test_create.py | 254 +++++++ .../zarr-metadata/tests/model/test_array.py | 29 +- .../zarr-metadata/tests/rules/__init__.py | 0 .../tests/rules/test_documents.py | 127 ++++ .../tests/rules/test_registry.py | 172 +++++ .../zarr-metadata/tests/rules/test_result.py | 107 +++ .../tests/rules/test_rule_properties.py | 171 +++++ .../tests/rules/test_spec_propagation.py | 160 +++++ .../tests/rules/test_v3_array_rules.py | 553 +++++++++++++++ .../zarr-metadata/tests/test_public_api.py | 24 +- .../tests/test_registry_drift.py | 85 +++ .../zarr-metadata/tests/v3/codec/test_kind.py | 26 + .../tests/v3/test_extension_points.py | 76 ++ .../tests/v3/test_shape_properties.py | 51 ++ 55 files changed, 5681 insertions(+), 74 deletions(-) create mode 100644 packages/zarr-metadata/changes/296.feature.3.md create mode 100644 packages/zarr-metadata/changes/296.feature.5.md create mode 100644 packages/zarr-metadata/changes/296.feature.6.md create mode 100644 packages/zarr-metadata/changes/296.feature.md create mode 100644 packages/zarr-metadata/docs/api/builder.md create mode 100644 packages/zarr-metadata/docs/api/rules.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/builder/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/builder/_create.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_documents.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_engine.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_registry.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_result.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_spec.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_shape.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py create mode 100644 packages/zarr-metadata/tests/builder/__init__.py create mode 100644 packages/zarr-metadata/tests/builder/test_create.py create mode 100644 packages/zarr-metadata/tests/rules/__init__.py create mode 100644 packages/zarr-metadata/tests/rules/test_documents.py create mode 100644 packages/zarr-metadata/tests/rules/test_registry.py create mode 100644 packages/zarr-metadata/tests/rules/test_result.py create mode 100644 packages/zarr-metadata/tests/rules/test_rule_properties.py create mode 100644 packages/zarr-metadata/tests/rules/test_spec_propagation.py create mode 100644 packages/zarr-metadata/tests/rules/test_v3_array_rules.py create mode 100644 packages/zarr-metadata/tests/test_registry_drift.py create mode 100644 packages/zarr-metadata/tests/v3/codec/test_kind.py create mode 100644 packages/zarr-metadata/tests/v3/test_extension_points.py create mode 100644 packages/zarr-metadata/tests/v3/test_shape_properties.py diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 6b6b172aec..0a0aa89ec1 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -24,21 +24,27 @@ Two layers and an optional integration: ## What this is for The public `TypedDict` definitions describe the static JSON shape of Zarr -metadata. For strict, loc-aware validation of JSON loaded from disk, use the -model parser: +metadata. To judge JSON loaded from disk, structure and composition together, +use the rules layer; to get a normalized document model, use the model parser: ```python import json from zarr_metadata.model import ZarrV3ArrayMetadata +from zarr_metadata.rules import parse_array_metadata_v3 with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = ZarrV3ArrayMetadata.from_json(raw) +document = parse_array_metadata_v3(raw) # raises with every problem found +metadata = ZarrV3ArrayMetadata.from_json(document) ``` -The optional Pydantic integration delegates raw input to the same strict -parser and returns the same normalized model class: +To construct a document, the `create_*` factories in `zarr_metadata.builder` +apply the same judgment to keyword arguments typed by the document's +`TypedDict`. + +The optional Pydantic integration runs raw input through the rules layer +and returns the same normalized model class: ```python from pydantic import TypeAdapter @@ -56,11 +62,19 @@ members that the strict model parser rejects. The model validators enforce the declared document structure and a small set of context-free consistency rules, including fixed format literals, finite -JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one -`dimension_names` entry per array dimension. They do not interpret extension -names or configurations, resolve codec pipelines, or decide whether a data -type, chunk grid, codec, or storage transformer is supported. Those decisions -belong to consumer implementations. +JSON numbers, non-negative dimensions, and non-empty v3 codec pipelines. +They do not interpret extension names or configurations. + +The composition rules (`zarr_metadata.rules`) judge the document as a whole: +fill values against data types, codec pipeline ordering, chunk-grid +geometry against `shape`, one `dimension_names` entry per array dimension, +and the canonical configuration shapes of the codecs, chunk grids, chunk +key encodings, and data types this package defines. Unknown extension names +are left unjudged. The rules model canonical documents and are deliberately +stricter than any given implementation: an implementation may coerce +ambiguous input as it sees fit and then validate the canonical result. +Nothing here decides whether a data type, chunk grid, codec, or storage +transformer is *supported*; that belongs to consumer implementations. The Pydantic integration's generated JSON Schemas express independently checkable document structure and field constraints, but they are not a @@ -68,7 +82,8 @@ replacement for runtime model validation. Standard JSON Schema treats a mathematically integral number such as `1.0` as an integer, while the runtime boundary requires Python `int` values, and it cannot express arbitrary same-length relations such as `dimension_names` versus `shape` or v2 `chunks` -versus `shape`. Consumers should run the model parser after schema validation. +versus `shape`. Consumers should run the runtime validators after schema +validation. ## Scope diff --git a/packages/zarr-metadata/changes/296.feature.3.md b/packages/zarr-metadata/changes/296.feature.3.md new file mode 100644 index 0000000000..f699ac068f --- /dev/null +++ b/packages/zarr-metadata/changes/296.feature.3.md @@ -0,0 +1,14 @@ +Added `create_*` factories in `zarr_metadata.builder`, one per public +document TypedDict (`create_zarr_v3_array_metadata_json`, +`create_zarr_v3_group_metadata_json`, `create_zarr_v3_consolidated_metadata_json`, +`create_zarr_v2_array_metadata_json`, `create_zarr_v2_group_metadata_json`, +`create_zarr_v2_zarray_json`, `create_zarr_v2_zgroup_json`, +`create_zarr_v2_consolidated_metadata_json`), each taking +`**kwargs: Unpack[]`. Each factory copies and normalizes its +input, runs structural and composition validation, and raises one +`MetadataValidationError` containing all problems. The strict on-disk +`.zarray`/`.zgroup` factories reject `attributes` at runtime, and the v2 +consolidated factory validates each entry against the document shape its +path suffix selects. The open v3 array/group factories take an +`extensions=` mapping for extension fields (for type checkers without PEP +728 support) and reject names that shadow standard fields. diff --git a/packages/zarr-metadata/changes/296.feature.5.md b/packages/zarr-metadata/changes/296.feature.5.md new file mode 100644 index 0000000000..1a5433c683 --- /dev/null +++ b/packages/zarr-metadata/changes/296.feature.5.md @@ -0,0 +1,17 @@ +Added `check_*` entry points in `zarr_metadata.rules` returning a +discriminated `Valid[T] | Invalid`, for callers who want a document and +its problems in one value. + +The literal `valid` field narrows to either the normalized document or a +nonempty problem tuple: + +```python +result = check_array_metadata_v3(loaded) +if result.valid: + store(result.document) # typed ZarrV3ArrayMetadataJSON +else: + report(result.problems) # non-empty tuple of problems +``` + +Use `validate_*` to collect problems and `parse_*` to raise on invalid +input. diff --git a/packages/zarr-metadata/changes/296.feature.6.md b/packages/zarr-metadata/changes/296.feature.6.md new file mode 100644 index 0000000000..fe9d67eced --- /dev/null +++ b/packages/zarr-metadata/changes/296.feature.6.md @@ -0,0 +1,13 @@ +Unknown members inside a *known* entity's `configuration` (e.g. an extra +key in a `blosc` configuration) now report as their own `unknown_key` +problem kind, and no longer suppress the other rules about that entity. + +Whether configurations are closed remains unspecified +([zarr-specs#270](https://github.com/zarr-developers/zarr-specs/issues/270)), +so this package retains its strict reading with two safeguards: + +- callers can filter the dedicated `unknown_key` kind; +- unknown keys do not suppress other rules for the same entity. + +Model round-trips preserve unmodeled members. Shape-exact `TypeIs` guards +still reject them because the corresponding TypedDicts are closed. diff --git a/packages/zarr-metadata/changes/296.feature.md b/packages/zarr-metadata/changes/296.feature.md new file mode 100644 index 0000000000..0e8c5b0e41 --- /dev/null +++ b/packages/zarr-metadata/changes/296.feature.md @@ -0,0 +1,51 @@ +Added `zarr_metadata.rules`: composition rules for full metadata +documents. The package now models metadata in three layers with one +contract each — `model` checks structure element by element, `rules` +judges composition across the document, and `builder` constructs while +applying both. Rules are registered where they are defined; rules about a +particular codec, chunk grid, or data type live with that entity under +`rules._entities` and are dispatched by name, so adding an entity adds a +module there and changes nothing else. + +- **Rule sets**: `ZARR_V3_ARRAY_RULES` covers fill value vs. data type, + codec pipeline kind ordering, known-name shapes, + dimension-name counts, chunk-grid values (positive extents) and + geometry (regular rank; rectilinear rank and per-dimension chunk-size + sums, RLE pairs included), transpose orders (self-permutation at any + depth, rank agreement with `shape`), and sharding (inner `codecs` and + `index_codecs` judged as pipelines recursively at every nesting depth; + inner chunk shapes positive, rank-matched, and evenly dividing the + enclosing chunk, recursively). New `ZARR_V2_ARRAY_RULES` + (chunks/shape rank agreement) and `ZARR_V3_GROUP_RULES` (inline + consolidated metadata recurses, judging each embedded child document + by its own rules at its path). +- **Read-side trios**: `validate_*` / `is_*` / `parse_*` for array and + group documents in both format versions mirror the model layer's + grammar with a stronger judgment — structure *and* composition, every + problem reported together, JSON arrays normalized to tuples before + judgment. The `is_*` functions deliberately return `bool` rather than + `TypeIs`: a composition-invalid document is still an instance of the + TypedDict, so only the structural layer can narrow honestly. +- **Boundary change**: two composition checks that lived in the + structural validator moved here — v3 `dimension_names` vs `shape` and + v2 `chunks` vs `shape` rank agreement. `zarr_metadata.model`'s + validators, parsers, and dataclasses now accept those documents (they + are lossless, structurally well-formed representations of what a store + may contain); use the `rules` trios to judge them. This also removes + the double report the overlap used to produce. +- **Strictness stance**, now documented on the package: `zarr_metadata` + models canonical documents and is deliberately stricter than any given + implementation; implementations coerce ambiguous input as they see fit + and then validate the canonical result. + +- **Codec chains are judged against the array each codec receives**: + `transpose` permutes the shape and `cast_value` changes the data type + seen by everything after it, so a shard behind a transpose must divide + the transposed chunk, and a `bytes` codec behind a cast needs an + endianness for the *target* type. `zarr_metadata.v3.codec.kind` sorts + known codec names into the spec's three pipeline kinds. +- **Pydantic field types** for array and group documents now run the + composition rules as well as structural validation. + +Known follow-up: v2 fill-value/dtype consistency (NumPy dtype grammar) +has no rule yet. diff --git a/packages/zarr-metadata/docs/api/builder.md b/packages/zarr-metadata/docs/api/builder.md new file mode 100644 index 0000000000..50bf9ac278 --- /dev/null +++ b/packages/zarr-metadata/docs/api/builder.md @@ -0,0 +1,5 @@ +--- +title: builder +--- + +::: zarr_metadata.builder diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 5e230c7aa2..1a73ec3842 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -8,6 +8,12 @@ The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.model`](model.md) — frozen-dataclass document models, structural validators, loc-aware parsers, and the `UNSET` sentinel +- [`zarr_metadata.rules`](rules.md) — composition rules: cross-field + judgments over full documents (fill value vs. data type, codec pipeline + ordering, chunk geometry), plus whole-document `validate`/`is`/`parse` + trios combining structure and composition +- [`zarr_metadata.builder`](builder.md) — validated construction: + one-shot `create_*` factories, one per document type - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents diff --git a/packages/zarr-metadata/docs/api/rules.md b/packages/zarr-metadata/docs/api/rules.md new file mode 100644 index 0000000000..ef010b72e2 --- /dev/null +++ b/packages/zarr-metadata/docs/api/rules.md @@ -0,0 +1,5 @@ +--- +title: rules +--- + +::: zarr_metadata.rules diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index 0163dda5c1..2368094796 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -71,11 +71,19 @@ members that the strict model parser rejects. The model validators enforce the declared document structure and a small set of context-free consistency rules, including fixed format literals, finite -JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one -`dimension_names` entry per array dimension. They do not interpret extension -names or configurations, resolve codec pipelines, or decide whether a data -type, chunk grid, codec, or storage transformer is supported. Those decisions -belong to consumer implementations. +JSON numbers, non-negative dimensions, and non-empty v3 codec pipelines. +They do not interpret extension names or configurations. + +The composition rules (`zarr_metadata.rules`) judge the document as a whole: +fill values against data types, codec pipeline ordering, chunk-grid +geometry against `shape`, one `dimension_names` entry per array dimension, +and the canonical configuration shapes of the codecs, chunk grids, chunk +key encodings, and data types this package defines. Unknown extension names +are left unjudged. The rules model canonical documents and are deliberately +stricter than any given implementation: an implementation may coerce +ambiguous input as it sees fit and then validate the canonical result. +Nothing here decides whether a data type, chunk grid, codec, or storage +transformer is *supported*; that belongs to consumer implementations. ## Scope diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 98a51e3645..225fb600e4 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -16,6 +16,8 @@ nav: - API Reference: - api/index.md - ' zarr_metadata.model': api/model.md + - ' zarr_metadata.rules': api/rules.md + - ' zarr_metadata.builder': api/builder.md - ' zarr_metadata.pydantic': api/pydantic.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 6e7b0e4f52..814eeccef4 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://zarr-metadata.readthedocs.io/" [dependency-groups] -test = ["pytest", "pydantic>=2.13", "jsonschema"] +test = ["pytest", "pydantic>=2.13", "jsonschema", "hypothesis"] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so the two sites render with the same toolchain. diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py b/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py new file mode 100644 index 0000000000..10dcb0dfe9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py @@ -0,0 +1,32 @@ +"""Validated construction of Zarr metadata documents. + +`create_*` factories provide typed one-shot construction for every +document TypedDict: required keys and value types are checked statically +at literal-keyword call sites, and the runtime pass normalizes the input +and applies structural and composition validation, raising one +`MetadataValidationError` carrying every problem. + +Use `zarr_metadata.rules` to validate documents read from storage. +""" + +from zarr_metadata.builder._create import ( + create_zarr_v2_array_metadata_json, + create_zarr_v2_consolidated_metadata_json, + create_zarr_v2_group_metadata_json, + create_zarr_v2_zarray_json, + create_zarr_v2_zgroup_json, + create_zarr_v3_array_metadata_json, + create_zarr_v3_consolidated_metadata_json, + create_zarr_v3_group_metadata_json, +) + +__all__ = [ + "create_zarr_v2_array_metadata_json", + "create_zarr_v2_consolidated_metadata_json", + "create_zarr_v2_group_metadata_json", + "create_zarr_v2_zarray_json", + "create_zarr_v2_zgroup_json", + "create_zarr_v3_array_metadata_json", + "create_zarr_v3_consolidated_metadata_json", + "create_zarr_v3_group_metadata_json", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/_create.py b/packages/zarr-metadata/src/zarr_metadata/builder/_create.py new file mode 100644 index 0000000000..58aac4c47b --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/builder/_create.py @@ -0,0 +1,333 @@ +"""One-shot factories for metadata document TypedDicts. + +Each factory copies and normalizes its input, then applies structural and +composition validation. Invalid input raises one +`MetadataValidationError`. V3 array and group factories accept extension +fields through `extensions=` for compatibility with type checkers that do +not support PEP 728. + +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, cast + +from typing_extensions import Unpack + +from zarr_metadata.model._group import ZarrV2ConsolidatedMetadata +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + GROUP_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_consolidated_metadata_v3, +) +from zarr_metadata.rules import ( + ZARR_V2_ARRAY_RULES, + ZARR_V3_ARRAY_RULES, + ZARR_V3_GROUP_RULES, + run_rules, + validate_array_metadata_v2, + validate_group_metadata_v2, +) +from zarr_metadata.rules._v3_group import consolidated_entries_problems + +if TYPE_CHECKING: + from collections.abc import Set as AbstractSet + + from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ZArrayJSON + from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2ZGroupJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField + from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + + +def _merged_with_extensions( + kwargs: Mapping[str, object], + extensions: Mapping[str, ZarrV3ExtensionField] | None, + standard_keys: AbstractSet[str], +) -> tuple[dict[str, object], list[ValidationProblem]]: + """`kwargs` plus `extensions`, refusing extension names that shadow standard keys. + + A colliding name is reported and *not* merged, so a later structural or + semantic pass judges the standard field the caller actually passed + rather than a value smuggled in through the extension hatch. + """ + document = dict(kwargs) + problems: list[ValidationProblem] = [] + for name, value in (extensions or {}).items(): + if name in standard_keys: + problems.append( + ValidationProblem( + (name,), + f"{name!r} is a standard metadata key; pass it as a keyword argument", + "invalid_value", + ) + ) + else: + document[name] = value + return document, problems + + +def _normalized(document: Mapping[str, object]) -> dict[str, object]: + """A deep copy of `document` with JSON arrays materialized as tuples. + + Deep-copying first means the returned document shares no mutable state + with the caller's arguments: mutating an input after the factory + returns cannot alter the validated result. + """ + return cast("dict[str, object]", arrays_to_tuples(copy.deepcopy(dict(document)))) + + +def _raise_if_problems(problems: Sequence[ValidationProblem]) -> None: + if len(problems) != 0: + raise MetadataValidationError(problems) + + +def _reject_attributes(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Problems for an `attributes` key in a strict on-disk v2 document. + + The strict `.zarray` / `.zgroup` shapes exclude `attributes` (it lives + in the sibling `.zattrs` file). The signature enforces that statically + at keyword call sites; this is the runtime backstop for `**`-splatted + and untyped callers, without which the merged-form parser would accept + the key and the returned value would not be the type it claims. + """ + if "attributes" not in document: + return () + return ( + ValidationProblem( + ("attributes",), + "'attributes' is not part of the on-disk document (it belongs to the " + "sibling .zattrs file); use the merged-form factory instead", + "invalid_value", + ), + ) + + +def create_zarr_v3_array_metadata_json( + *, + extensions: Mapping[str, ZarrV3ExtensionField] | None = None, + **kwargs: Unpack[ZarrV3ArrayMetadataJSON], +) -> ZarrV3ArrayMetadataJSON: + """A validated v3 array metadata document (the `zarr.json` content for an array). + + Required keys are enforced statically by the signature; at runtime the + document is checked structurally (via the model layer's parser) and + semantically (via `ZARR_V3_ARRAY_RULES`), and every problem from both + passes is raised together in one `MetadataValidationError`. Extension + fields go in `extensions`; names that shadow standard keys are rejected. + """ + document, problems = _merged_with_extensions( + kwargs, extensions, ARRAY_METADATA_STANDARD_KEYS_V3 + ) + normalized = _normalized(document) + parsed: ZarrV3ArrayMetadataJSON | None = None + try: + parsed = parse_array_metadata_v3(normalized) + except MetadataValidationError as error: + problems.extend(error.problems) + problems.extend(run_rules(ZARR_V3_ARRAY_RULES, normalized)) + _raise_if_problems(problems) + assert parsed is not None + return parsed + + +def create_zarr_v3_group_metadata_json( + *, + extensions: Mapping[str, ZarrV3ExtensionField] | None = None, + **kwargs: Unpack[ZarrV3GroupMetadataJSON], +) -> ZarrV3GroupMetadataJSON: + """A validated v3 group metadata document (the `zarr.json` content for a group). + + Extension fields go in `extensions`; names that shadow standard keys + are rejected. The composition rules recurse into inline consolidated + metadata, so an embedded child document invalid under its own rules + is reported here, at its path. + """ + document, problems = _merged_with_extensions( + kwargs, extensions, GROUP_METADATA_STANDARD_KEYS_V3 + ) + normalized = _normalized(document) + parsed: ZarrV3GroupMetadataJSON | None = None + try: + parsed = parse_group_metadata_v3(normalized) + except MetadataValidationError as error: + problems.extend(error.problems) + problems.extend(run_rules(ZARR_V3_GROUP_RULES, normalized)) + _raise_if_problems(problems) + assert parsed is not None + return parsed + + +def create_zarr_v3_consolidated_metadata_json( + **kwargs: Unpack[ZarrV3ConsolidatedMetadataJSON], +) -> ZarrV3ConsolidatedMetadataJSON: + """A validated v3 inline consolidated metadata object. + + This is the value embedded in a v3 group document under the + `consolidated_metadata` key, not a store document of its own. + """ + normalized = _normalized(kwargs) + _raise_if_problems( + validate_consolidated_metadata_v3(normalized) + consolidated_entries_problems(normalized) + ) + return cast("ZarrV3ConsolidatedMetadataJSON", normalized) + + +def create_zarr_v2_array_metadata_json( + **kwargs: Unpack[ZarrV2ArrayMetadataJSON], +) -> ZarrV2ArrayMetadataJSON: + """A validated v2 array metadata document, in-memory merged form. + + Models `.zarray` plus the sibling `.zattrs` folded in as `attributes`. + For the strict on-disk `.zarray` shape use `create_zarr_v2_zarray_json`. + """ + normalized = _normalized(kwargs) + parsed: ZarrV2ArrayMetadataJSON | None = None + problems: list[ValidationProblem] = [] + try: + parsed = parse_array_metadata_v2(normalized) + except MetadataValidationError as error: + problems.extend(error.problems) + problems.extend(run_rules(ZARR_V2_ARRAY_RULES, normalized)) + _raise_if_problems(problems) + assert parsed is not None + return parsed + + +def create_zarr_v2_group_metadata_json( + **kwargs: Unpack[ZarrV2GroupMetadataJSON], +) -> ZarrV2GroupMetadataJSON: + """A validated v2 group metadata document, in-memory merged form. + + Models `.zgroup` plus the sibling `.zattrs` folded in as `attributes`. + For the strict on-disk `.zgroup` shape use `create_zarr_v2_zgroup_json`. + """ + parsed: ZarrV2GroupMetadataJSON | None = None + problems: list[ValidationProblem] = [] + try: + parsed = parse_group_metadata_v2(_normalized(kwargs)) + except MetadataValidationError as error: + problems.extend(error.problems) + _raise_if_problems(problems) + assert parsed is not None + return parsed + + +def create_zarr_v2_zarray_json(**kwargs: Unpack[ZarrV2ZArrayJSON]) -> ZarrV2ZArrayJSON: + """A validated on-disk `.zarray` document (strict form, no `attributes`). + + Structurally checked with the merged-form parser plus a runtime + rejection of `attributes`: the strict shape is the merged shape minus + `attributes`, and the runtime check holds for callers the signature's + static exclusion cannot see (`**`-splatted mappings, untyped code). + """ + normalized = _normalized(kwargs) + problems: list[ValidationProblem] = list(_reject_attributes(normalized)) + parsed: ZarrV2ArrayMetadataJSON | None = None + try: + parsed = parse_array_metadata_v2(normalized) + except MetadataValidationError as error: + problems.extend(error.problems) + problems.extend(run_rules(ZARR_V2_ARRAY_RULES, normalized)) + _raise_if_problems(problems) + assert parsed is not None + return cast("ZarrV2ZArrayJSON", parsed) + + +def create_zarr_v2_zgroup_json(**kwargs: Unpack[ZarrV2ZGroupJSON]) -> ZarrV2ZGroupJSON: + """A validated on-disk `.zgroup` document (strict form, no `attributes`). + + Structurally checked with the merged-form parser plus a runtime + rejection of `attributes`: the strict shape is the merged shape minus + `attributes`, and the runtime check holds for callers the signature's + static exclusion cannot see (`**`-splatted mappings, untyped code). + """ + normalized = _normalized(kwargs) + problems: list[ValidationProblem] = list(_reject_attributes(normalized)) + parsed: ZarrV2GroupMetadataJSON | None = None + try: + parsed = parse_group_metadata_v2(normalized) + except MetadataValidationError as error: + problems.extend(error.problems) + _raise_if_problems(problems) + assert parsed is not None + return cast("ZarrV2ZGroupJSON", parsed) + + +def _validate_v2_consolidated_envelope( + document: Mapping[str, object], +) -> tuple[ValidationProblem, ...]: + """Every reason `document` is not a `.zmetadata` envelope. + + The envelope itself is the model layer's judgment; on top of it, each + entry's path suffix selects the strict on-disk document shape that its + value must satisfy. + """ + try: + ZarrV2ConsolidatedMetadata.from_json(document) + except MetadataValidationError as error: + return error.problems + problems: list[ValidationProblem] = [] + for key, entry in cast("Mapping[str, object]", document["metadata"]).items(): + if not isinstance(entry, Mapping): + problems.append( + ValidationProblem(("metadata", key), "expected a JSON object", "invalid_type") + ) + continue + entry_mapping = cast("Mapping[str, object]", entry) + if key.endswith(".zarray"): + nested = validate_array_metadata_v2(entry_mapping) + _reject_attributes(entry_mapping) + elif key.endswith(".zgroup"): + nested = validate_group_metadata_v2(entry_mapping) + _reject_attributes(entry_mapping) + elif key.endswith(".zattrs"): + nested = () + else: + nested = ( + ValidationProblem( + (), + "expected a v2 metadata file suffix: .zarray, .zgroup, or .zattrs", + "invalid_value", + ), + ) + problems.extend( + ValidationProblem(("metadata", key, *found.loc), found.message, found.kind) + for found in nested + ) + return tuple(problems) + + +def create_zarr_v2_consolidated_metadata_json( + **kwargs: Unpack[ZarrV2ConsolidatedMetadataJSON], +) -> ZarrV2ConsolidatedMetadataJSON: + """A validated `.zmetadata` consolidated metadata document. + + The runtime pass checks the envelope and validates each nested value + against the strict document shape selected by its path suffix. This is + the runtime backstop for callers the signature's static enforcement + cannot see (`**`-splatted mappings, untyped code). + """ + normalized = _normalized(kwargs) + _raise_if_problems(_validate_v2_consolidated_envelope(normalized)) + return cast("ZarrV2ConsolidatedMetadataJSON", normalized) + + +__all__ = [ + "create_zarr_v2_array_metadata_json", + "create_zarr_v2_consolidated_metadata_json", + "create_zarr_v2_group_metadata_json", + "create_zarr_v2_zarray_json", + "create_zarr_v2_zgroup_json", + "create_zarr_v3_array_metadata_json", + "create_zarr_v3_consolidated_metadata_json", + "create_zarr_v3_group_metadata_json", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index f927851e4c..95c0822c59 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -7,8 +7,8 @@ Every `ValidationProblem` carries a machine-readable `kind` alongside its human-readable `message`, so consumers can dispatch on the failure mode -(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`) without -string-matching messages. +(`missing_key`, `invalid_type`, `invalid_value`, `invalid_json`, +`unknown_key`) without string-matching messages. """ from __future__ import annotations @@ -28,7 +28,7 @@ from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON -ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json"] +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] """Machine-readable classification of a `ValidationProblem`. - `missing_key`: a required key (document key or store key) is absent. @@ -37,6 +37,15 @@ - `invalid_value`: a value has an acceptable type but an invalid content (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). - `invalid_json`: bytes that do not decode as JSON. +- `unknown_key`: a member this package does not model appears inside an + entity whose shape it does model (e.g. an extra key in a `blosc` + configuration). Distinguished from `invalid_value` because the Zarr v3 + spec does not say whether a `configuration` is closed + (zarr-developers/zarr-specs#270 has been open since 2023), so this is + the package's strict reading rather than a definite violation: a + document carrying one is very likely fine, just written by something + that models more than we do. Callers that prefer tolerance can filter + this kind out; the package itself never lets it mask other findings. """ @@ -550,16 +559,8 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: ("dimension_names",), "expected items of str or None", "invalid_type" ) ) - elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len( - cast("Sequence[int]", doc["shape"]) - ): - problems.append( - ValidationProblem( - ("dimension_names",), - "expected one name per dimension of shape", - "invalid_value", - ) - ) + # Whether the names count matches shape's dimensionality is a + # composition judgment, owned by zarr_metadata.rules. return tuple(problems) @@ -599,26 +600,10 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: # ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. problems: list[ValidationProblem] = list(_missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc)) problems.extend(_check_literal(doc, "zarr_format", 2)) - shape_problems = _validate_dim_sequence(doc, "shape") - chunks_problems = _validate_dim_sequence(doc, "chunks") - problems.extend(shape_problems) - problems.extend(chunks_problems) - if ( - len(shape_problems) == 0 - and len(chunks_problems) == 0 - and _is_int_sequence(doc.get("shape")) - and _is_int_sequence(doc.get("chunks")) - ): - shape = cast("Sequence[int]", doc["shape"]) - chunks = cast("Sequence[int]", doc["chunks"]) - if len(shape) != len(chunks): - problems.append( - ValidationProblem( - ("chunks",), - "expected the same number of dimensions as shape", - "invalid_value", - ) - ) + problems.extend(_validate_dim_sequence(doc, "shape")) + problems.extend(_validate_dim_sequence(doc, "chunks")) + # Whether chunks matches shape's dimensionality is a composition + # judgment, owned by zarr_metadata.rules. if "dtype" in doc and not _is_dtype_v2(doc["dtype"]): problems.append( ValidationProblem( diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 5b0c9e5b57..11d27d0a7e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -6,13 +6,17 @@ Each exported name is an `Annotated` field type over the corresponding core model class — the instances ARE the core classes, so values interoperate freely with non-pydantic code (equality, isinstance, nesting). Validation -delegates to the library: a raw document routes through `from_json` (the -single source of truth for structural validation and normalization, so -pydantic's field-level coercion can never bypass it), an existing model -instance passes through unchanged, and serialization emits the canonical -document via `to_json`. `MetadataValidationError` subclasses `ValueError`, -so a failed parse surfaces as a pydantic `ValidationError` carrying the -loc-annotated problem messages. +delegates to the library: a raw document is judged by `zarr_metadata.rules` +(structure and composition together) and then normalized through +`from_json`, so pydantic's field-level coercion can never bypass either +layer; an existing model instance passes through unchanged, and +serialization emits the canonical document via `to_json`. +`MetadataValidationError` subclasses `ValueError`, so a failed parse +surfaces as a pydantic `ValidationError` carrying the loc-annotated +problem messages. + +The v2 consolidated field type and the bare metadata-field type carry no +composition rules and are validated structurally. Usage: @@ -33,6 +37,7 @@ class ArrayManifest(BaseModel): from pydantic import BeforeValidator, InstanceOf, PlainSerializer from zarr_metadata import model as _model +from zarr_metadata import rules as _rules from zarr_metadata._pydantic_schema import ( ZarrV2ArrayMetadataJSON as _ZarrV2ArrayMetadataSchema, ) @@ -54,6 +59,8 @@ class ArrayManifest(BaseModel): from zarr_metadata._pydantic_schema import ( ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, ) +from zarr_metadata.model._validation import arrays_to_tuples, validate_consolidated_metadata_v3 +from zarr_metadata.rules._v3_group import consolidated_entries_problems if TYPE_CHECKING: from collections.abc import Callable @@ -72,10 +79,33 @@ def coerce(value: object) -> _M: return coerce +def _judged_by( + validate: Callable[[object], tuple[_model.ValidationProblem, ...]], + parse: Callable[[object], _M], +) -> Callable[[object], _M]: + """`parse`, preceded by a whole-document judgment that raises on any problem.""" + + def judged(value: object) -> _M: + problems = validate(value) + if len(problems) != 0: + raise _model.MetadataValidationError(problems) + return parse(value) + + return judged + + +def _validate_consolidated_v3(value: object) -> tuple[_model.ValidationProblem, ...]: + normalized = arrays_to_tuples(value) + return validate_consolidated_metadata_v3(normalized) + consolidated_entries_problems(normalized) + + ZarrV3ArrayMetadata = Annotated[ InstanceOf[_model.ZarrV3ArrayMetadata], BeforeValidator( - _coerce_to(_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), + _coerce_to( + _model.ZarrV3ArrayMetadata, + _judged_by(_rules.validate_array_metadata_v3, _model.ZarrV3ArrayMetadata.from_json), + ), json_schema_input_type=_ZarrV3ArrayMetadataSchema, ), PlainSerializer(_model.ZarrV3ArrayMetadata.to_json, return_type=_ZarrV3ArrayMetadataSchema), @@ -85,7 +115,10 @@ def coerce(value: object) -> _M: ZarrV2ArrayMetadata = Annotated[ InstanceOf[_model.ZarrV2ArrayMetadata], BeforeValidator( - _coerce_to(_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), + _coerce_to( + _model.ZarrV2ArrayMetadata, + _judged_by(_rules.validate_array_metadata_v2, _model.ZarrV2ArrayMetadata.from_json), + ), json_schema_input_type=_ZarrV2ArrayMetadataSchema, ), PlainSerializer(_model.ZarrV2ArrayMetadata.to_json, return_type=_ZarrV2ArrayMetadataSchema), @@ -95,7 +128,10 @@ def coerce(value: object) -> _M: ZarrV3GroupMetadata = Annotated[ InstanceOf[_model.ZarrV3GroupMetadata], BeforeValidator( - _coerce_to(_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), + _coerce_to( + _model.ZarrV3GroupMetadata, + _judged_by(_rules.validate_group_metadata_v3, _model.ZarrV3GroupMetadata.from_json), + ), json_schema_input_type=_ZarrV3GroupMetadataSchema, ), PlainSerializer(_model.ZarrV3GroupMetadata.to_json, return_type=_ZarrV3GroupMetadataSchema), @@ -105,7 +141,10 @@ def coerce(value: object) -> _M: ZarrV2GroupMetadata = Annotated[ InstanceOf[_model.ZarrV2GroupMetadata], BeforeValidator( - _coerce_to(_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), + _coerce_to( + _model.ZarrV2GroupMetadata, + _judged_by(_rules.validate_group_metadata_v2, _model.ZarrV2GroupMetadata.from_json), + ), json_schema_input_type=_ZarrV2GroupMetadataSchema, ), PlainSerializer(_model.ZarrV2GroupMetadata.to_json, return_type=_ZarrV2GroupMetadataSchema), @@ -117,7 +156,7 @@ def coerce(value: object) -> _M: BeforeValidator( _coerce_to( _model.ZarrV3ConsolidatedMetadata, - _model.ZarrV3ConsolidatedMetadata.from_json, + _judged_by(_validate_consolidated_v3, _model.ZarrV3ConsolidatedMetadata.from_json), ), json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, ), diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py new file mode 100644 index 0000000000..0631f838c8 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py @@ -0,0 +1,73 @@ +"""Validate structure and composition of Zarr metadata documents. + +`zarr_metadata.model` checks JSON structure. This module also checks +cross-field constraints such as fill-value compatibility, codec +ordering, and dimension counts. Its `validate_*`, `is_*`, and `parse_*` +functions mirror the model API; `check_*` returns `Valid[T] | Invalid`. + +Rules target canonical metadata and may be stricter than readers that +coerce inputs. Unknown entity names are left unjudged. Known entities +must match their modeled shape; extra configuration keys produce an +`unknown_key` problem without suppressing other checks. Model +round-trips preserve those unmodeled members. +""" + +from zarr_metadata.rules._documents import ( + is_array_metadata_v2, + is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + parse_group_metadata_v2, + parse_group_metadata_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) +from zarr_metadata.rules._engine import Rule, RuleCheck, applicable, run_rules +from zarr_metadata.rules._result import ( + Invalid, + Valid, + ValidationResult, + check_array_metadata_v2, + check_array_metadata_v3, + check_group_metadata_v2, + check_group_metadata_v3, +) +from zarr_metadata.rules._v2_array import ZARR_V2_ARRAY, ZARR_V2_ARRAY_RULES +from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY, ZARR_V3_ARRAY_RULES +from zarr_metadata.rules._v3_group import ZARR_V3_GROUP, ZARR_V3_GROUP_RULES + +__all__ = [ + "ZARR_V2_ARRAY", + "ZARR_V2_ARRAY_RULES", + "ZARR_V3_ARRAY", + "ZARR_V3_ARRAY_RULES", + "ZARR_V3_GROUP", + "ZARR_V3_GROUP_RULES", + "Invalid", + "Rule", + "RuleCheck", + "Valid", + "ValidationResult", + "applicable", + "check_array_metadata_v2", + "check_array_metadata_v3", + "check_group_metadata_v2", + "check_group_metadata_v3", + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", + "run_rules", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py new file mode 100644 index 0000000000..d88ff862d0 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -0,0 +1,187 @@ +"""Whole-document structural and composition validation. + +These `validate_*`, `is_*`, and `parse_*` functions mirror the model API +but apply both validation layers. The `is_*` functions return `bool`, not +`TypeIs`: composition validity is stricter than TypedDict membership. +Use `zarr_metadata.model.is_*` for type narrowing. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ( + MetadataValidationError, + arrays_to_tuples, +) +from zarr_metadata.model._validation import ( + validate_array_metadata_v2 as _validate_structure_v2, +) +from zarr_metadata.model._validation import ( + validate_array_metadata_v3 as _validate_structure_v3, +) +from zarr_metadata.model._validation import ( + validate_group_metadata_v2 as _validate_group_structure_v2, +) +from zarr_metadata.model._validation import ( + validate_group_metadata_v3 as _validate_group_structure_v3, +) +from zarr_metadata.rules._engine import run_rules +from zarr_metadata.rules._v2_array import ZARR_V2_ARRAY_RULES +from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY_RULES +from zarr_metadata.rules._v3_group import ZARR_V3_GROUP_RULES + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ValidationProblem + from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + + +def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not a valid v3 array document. + + Structural problems (from the model layer) and composition problems + (from `ZARR_V3_ARRAY_RULES`) are reported together. JSON arrays are + normalized to tuples before judgment, so list-spelled documents + (e.g. fresh `json.loads` output) are judged at the canonical data + level rather than rejected for their spelling. + """ + normalized = arrays_to_tuples(value) + problems = _validate_structure_v3(normalized) + if isinstance(normalized, Mapping): + document = cast("Mapping[str, object]", normalized) + problems = problems + run_rules(ZARR_V3_ARRAY_RULES, document) + return tuple(problems) + + +def is_array_metadata_v3(value: object) -> bool: + """Whether `value` is a structurally and compositionally valid v3 array doc. + + Deliberately not a `TypeIs` guard — see the module docstring. Use + `zarr_metadata.model.is_array_metadata_v3` to narrow. + """ + return len(validate_array_metadata_v3(value)) == 0 + + +def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: + """Return `value` as a valid `ZarrV3ArrayMetadataJSON`, or raise. + + Normalizes JSON arrays to tuples, then raises a single + `MetadataValidationError` carrying every structural and composition + problem found. + """ + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v3(normalized) + if len(problems) != 0: + raise MetadataValidationError(problems) + return cast("ZarrV3ArrayMetadataJSON", normalized) + + +def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not a valid v2 array document (merged form). + + JSON arrays are normalized to tuples before judgment, as in + `validate_array_metadata_v3`. + """ + normalized = arrays_to_tuples(value) + problems = _validate_structure_v2(normalized) + if isinstance(normalized, Mapping): + document = cast("Mapping[str, object]", normalized) + problems = problems + run_rules(ZARR_V2_ARRAY_RULES, document) + return tuple(problems) + + +def is_array_metadata_v2(value: object) -> bool: + """Whether `value` is a structurally and compositionally valid v2 array doc. + + Deliberately not a `TypeIs` guard — see the module docstring. + """ + return len(validate_array_metadata_v2(value)) == 0 + + +def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: + """Return `value` as a valid `ZarrV2ArrayMetadataJSON`, or raise. + + Normalizes JSON arrays to tuples, then raises a single + `MetadataValidationError` carrying every structural and composition + problem found. + """ + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v2(normalized) + if len(problems) != 0: + raise MetadataValidationError(problems) + return cast("ZarrV2ArrayMetadataJSON", normalized) + + +def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not a valid v3 group document. + + Composition rules recurse into inline consolidated metadata, so a + consolidated child document invalid under its own rules is reported + here, at its path. + """ + normalized = arrays_to_tuples(value) + problems = _validate_group_structure_v3(normalized) + if isinstance(normalized, Mapping): + document = cast("Mapping[str, object]", normalized) + problems = problems + run_rules(ZARR_V3_GROUP_RULES, document) + return tuple(problems) + + +def is_group_metadata_v3(value: object) -> bool: + """Whether `value` is a structurally and compositionally valid v3 group doc. + + Deliberately not a `TypeIs` guard — see the module docstring. + """ + return len(validate_group_metadata_v3(value)) == 0 + + +def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: + """Return `value` as a valid `ZarrV3GroupMetadataJSON`, or raise.""" + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v3(normalized) + if len(problems) != 0: + raise MetadataValidationError(problems) + return cast("ZarrV3GroupMetadataJSON", normalized) + + +def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not a valid v2 group document (merged form). + + v2 group documents carry no composition constraints today, so this is + the structural judgment, offered here for a uniform read-side API. + """ + return _validate_group_structure_v2(arrays_to_tuples(value)) + + +def is_group_metadata_v2(value: object) -> bool: + """Whether `value` is a valid v2 group document (merged form).""" + return len(validate_group_metadata_v2(value)) == 0 + + +def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: + """Return `value` as a valid `ZarrV2GroupMetadataJSON`, or raise.""" + normalized = arrays_to_tuples(value) + problems = validate_group_metadata_v2(normalized) + if len(problems) != 0: + raise MetadataValidationError(problems) + return cast("ZarrV2GroupMetadataJSON", normalized) + + +__all__ = [ + "is_array_metadata_v2", + "is_array_metadata_v3", + "is_group_metadata_v2", + "is_group_metadata_v3", + "parse_array_metadata_v2", + "parse_array_metadata_v3", + "parse_group_metadata_v2", + "parse_group_metadata_v3", + "validate_array_metadata_v2", + "validate_array_metadata_v3", + "validate_group_metadata_v2", + "validate_group_metadata_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py b/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py new file mode 100644 index 0000000000..bb101bd301 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py @@ -0,0 +1,115 @@ +"""Rules gated by the document fields they read. + +A `Rule` runs when every key in `requires` is present. The same rule set +therefore supports complete documents and partial builders without +imposing field order. + +Prior art +--------- +The gate is conventional, which is the point. Ecto's +`Ecto.Changeset.validate_change/3` invokes a validator "only if a change +for the given field exists", so one changeset serves both full inserts +and partial updates. Clojure spec's two-phase `s/keys` separates +required-key presence from key/value conformance precisely because "we +routinely deal with optional and partial data". Valibot's `partialCheck` +takes the paths a cross-field rule reads and runs it "whenever the +selected part of the data is valid". Presence-conditional rules-as-data +are JSON Schema's `dependentSchemas` / `dependentRequired` applicators. + +- https://hexdocs.pm/ecto/Ecto.Changeset.html +- https://clojure.org/about/spec +- https://valibot.dev/api/partialCheck/ +- https://www.learnjsonschema.com/2020-12/applicator/dependentschemas/ + +Two consequences follow from gating rather than ordering. + +**Order-free by construction.** No topological sort, so mutually +dependent rules are expressible — unlike Yup, whose equivalent `deps` +orders rules and therefore rejects cycles outright. + +**Absence is deliberately inexpressible.** A rule cannot ask whether a +field is missing: that is negation-as-failure, sound only under a +closed-world assumption, and a partially built document is an open world +where the key may still arrive. Required-key checks therefore stay in +structural validation — the same stratification Ecto +(`validate_required`), spec (`:req`), and JSON Schema (`required`) apply. + +Rules may receive structurally invalid values. A rule that cannot safely +interpret its inputs leaves the problem to structural validation. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Set as AbstractSet +from dataclasses import dataclass +from typing import cast + +from zarr_metadata.model._validation import ValidationProblem + +RuleCheck = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] +"""A rule's check: the whole document in, every problem it finds out.""" + + +@dataclass(frozen=True, slots=True) +class Rule: + """One composition check over a (possibly partial) metadata document. + + `requires` are the document keys the check reads; the rule fires only + when all of them are present. `check` receives the whole document (so + coupled fields are examined together) and returns every problem it + finds, empty when the rule passes. + """ + + requires: frozenset[str] + check: RuleCheck + + +def applicable(rules: Sequence[Rule], present: AbstractSet[str]) -> Iterator[Rule]: + """The subset of `rules` whose required keys are all present.""" + return (rule for rule in rules if rule.requires <= present) + + +def run_rules( + rules: Sequence[Rule], document: Mapping[str, object] +) -> tuple[ValidationProblem, ...]: + """Run every applicable rule over `document`, collecting all problems.""" + problems: list[ValidationProblem] = [] + for rule in applicable(rules, document.keys()): + problems.extend(rule.check(document)) + return tuple(problems) + + +def as_string_mapping(value: object) -> Mapping[str, object] | None: + """`value` as a string-keyed mapping, or None if it is not one.""" + if not isinstance(value, Mapping): + return None + mapping = cast("Mapping[object, object]", value) + if any(not isinstance(key, str) for key in mapping): + return None + return cast("Mapping[str, object]", mapping) + + +def as_sequence(value: object) -> Sequence[object] | None: + """`value` as a JSON-array-shaped sequence, or None if it is not one.""" + if isinstance(value, (list, tuple)): + return cast("Sequence[object]", value) + return None + + +def prefixed( + loc: tuple[str | int, ...], problems: Sequence[ValidationProblem] +) -> tuple[ValidationProblem, ...]: + """Re-base every problem's `loc` under `loc` (for nested documents).""" + return tuple( + ValidationProblem((*loc, *problem.loc), problem.message, problem.kind) + for problem in problems + ) + + +__all__ = [ + "Rule", + "RuleCheck", + "applicable", + "run_rules", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py new file mode 100644 index 0000000000..a1662ccc1f --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py @@ -0,0 +1,27 @@ +"""Per-entity composition rules, discovered automatically. + +Every module here owns the rules for one codec or chunk grid and +registers them with `zarr_metadata.rules._registry` at import time. +Adding a new entity means adding a module here and nothing else: this +package imports every sibling module on import, so there is no +registration list to update and no document-rule module to edit. + +That auto-discovery is the deliberate answer to two failure modes. A +hand-written registry lets a rule be defined and never registered, and a +hand-written import list lets a whole module be defined and never +imported; both produce rules that silently never run. `tests/rules/ +test_registry.py` closes the remaining gap by asserting that every codec +and chunk grid the package models is either registered here or listed as +deliberately rule-free. +""" + +from __future__ import annotations + +import importlib +import pkgutil + +for _module in pkgutil.iter_modules(__path__): + if not _module.name.startswith("_"): + importlib.import_module(f"{__name__}.{_module.name}") + +__all__: list[str] = [] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py new file mode 100644 index 0000000000..048ce33fc7 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py @@ -0,0 +1,116 @@ +"""Composition rules for the core ``bytes`` codec.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._engine import as_string_mapping +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import CODECS, DATA_TYPE +from zarr_metadata.v3._shape import blocking_problems, validate_known_entity_metadata +from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME +from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArraySpec + +_ARRAY_V3 = "zarr_v3_array" + +_SINGLE_BYTE = frozenset({"bool", "int8", "uint8"}) +_MULTI_BYTE = frozenset( + { + "int16", + "int32", + "int64", + "uint16", + "uint32", + "uint64", + "float16", + "float32", + "float64", + "complex64", + "complex128", + "numpy.datetime64", + "numpy.timedelta64", + } +) +_VARIABLE_LENGTH = frozenset({"bytes", "string"}) +_StorageClass = Literal["single_byte", "multi_byte", "variable_length"] + + +def _data_type_name(data_type: object) -> str | None: + if isinstance(data_type, str): + return data_type + mapping = as_string_mapping(data_type) + if mapping is None: + return None + name = mapping.get("name") + return name if isinstance(name, str) else None + + +def _storage_class(data_type: object) -> _StorageClass | None: + """Classify known data types by their raw byte representation.""" + name = _data_type_name(data_type) + if name in _SINGLE_BYTE or (name is not None and RAW_BYTES_NAME_PATTERN.fullmatch(name)): + return "single_byte" + if name in _MULTI_BYTE: + return "multi_byte" + if name in _VARIABLE_LENGTH: + return "variable_length" + if name != "struct": + return None + + envelope = as_string_mapping(data_type) + configuration = ( + as_string_mapping(envelope.get("configuration")) if envelope is not None else None + ) + fields = configuration.get("fields") if configuration is not None else None + if not isinstance(fields, tuple): + return None + classes: list[_StorageClass] = [] + for field in cast("tuple[object, ...]", fields): + field_mapping = as_string_mapping(field) + if field_mapping is None or "data_type" not in field_mapping: + return None + field_class = _storage_class(field_mapping["data_type"]) + if field_class is None: + return None + classes.append(field_class) + if "variable_length" in classes: + return "variable_length" + if "multi_byte" in classes: + return "multi_byte" + return "single_byte" + + +@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME) +def data_type_has_a_raw_byte_representation( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + if incoming.data_type is None: + return () + shape_verdict = validate_known_entity_metadata(DATA_TYPE, incoming.data_type) + if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0: + return () + storage_class = _storage_class(incoming.data_type) + if storage_class == "variable_length": + name = _data_type_name(incoming.data_type) + return ( + ValidationProblem( + (), + f"bytes codec is not compatible with variable-length data_type {name!r}", + "invalid_value", + ), + ) + if storage_class == "multi_byte" and "endian" not in configuration: + return ( + ValidationProblem( + ("endian",), + "endian is required for a data type containing multi-byte values", + "missing_key", + ), + ) + return () diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py new file mode 100644 index 0000000000..99f2964eb4 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py @@ -0,0 +1,32 @@ +"""Spec transition for the `cast_value` codec. + +`cast_value` carries no composition rules of its own today, but it +changes the data type everything downstream receives: a later rule that +reads the type (the `bytes` codec's endianness requirement, for example) +must judge against the configured target. + +The codec also casts the fill value, and the spec makes a failed +round-trip a MUST error. Deciding that means implementing the cast +(rounding modes, out-of-range clamp and wrap, scalar maps), which is +numeric semantics rather than JSON judgment; it belongs to whatever +implements the codec and is not modelled here. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.rules._spec import ArraySpec, spec_transition +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + + +@spec_transition(CAST_VALUE_CODEC_NAME) +def cast_data_type(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: + """The outgoing type is the configured target.""" + target = cast("ZarrV3MetadataFieldJSON", configuration["data_type"]) + return incoming.with_data_type(target) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py new file mode 100644 index 0000000000..542574d1ad --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py @@ -0,0 +1,31 @@ +"""Composition rules for the core ``gzip`` codec.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import CODECS +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArraySpec + +_ARRAY_V3 = "zarr_v3_array" + + +@entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME) +def level_is_in_range( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + level = cast("int", configuration["level"]) + if 0 <= level <= 9: + return () + return ( + ValidationProblem( + ("level",), f"expected an integer in [0, 9], got {level}", "invalid_value" + ), + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py new file mode 100644 index 0000000000..c5baed7d4e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py @@ -0,0 +1,38 @@ +"""Composition rules shared by NumPy datetime and timedelta data types.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import DATA_TYPE +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArraySpec + +_ARRAY_V3 = "zarr_v3_array" +_MAX_SCALE_FACTOR = 2**31 - 1 + + +def _scale_factor_is_in_range( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + scale_factor = cast("int", configuration["scale_factor"]) + if 1 <= scale_factor <= _MAX_SCALE_FACTOR: + return () + return ( + ValidationProblem( + ("scale_factor",), + f"expected an integer in [1, {_MAX_SCALE_FACTOR}], got {scale_factor}", + "invalid_value", + ), + ) + + +entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE_NAME)(_scale_factor_is_in_range) +entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE_NAME)(_scale_factor_is_in_range) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py new file mode 100644 index 0000000000..15f170ee97 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py @@ -0,0 +1,125 @@ +"""Composition rules for the `rectilinear` chunk grid.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import CHUNK_GRID +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArraySpec, Sequence + +_ARRAY_V3 = "zarr_v3_array" + + +def _is_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _expanded_extent(spec: Sequence[object]) -> int | None: + """The total extent an explicit dimension spec covers, or None. + + Entries are chunk sizes or `[size, count]` run-length pairs. Answers + None when any entry is non-positive — the values rule owns that + complaint, and a sum over bad entries would be noise. + """ + total = 0 + for item in spec: + if _is_int(item) and cast(int, item) >= 1: + total += cast(int, item) + elif isinstance(item, tuple): + size, count = cast("tuple[object, object]", item) + if not (_is_int(size) and _is_int(count)): + return None + if cast(int, size) < 1 or cast(int, count) < 1: + return None + total += cast(int, size) * cast(int, count) + else: + return None + return total + + +@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME) +def chunk_extents_are_positive( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """Every chunk extent, bare or run-length encoded, must be positive.""" + chunk_shapes = cast("tuple[object, ...]", configuration["chunk_shapes"]) + problems: list[ValidationProblem] = [] + for dim, spec in enumerate(chunk_shapes): + loc: tuple[str | int, ...] = ("chunk_shapes", dim) + if _is_int(spec): + if cast(int, spec) < 1: + problems.append( + ValidationProblem( + loc, f"expected a positive chunk extent, got {spec}", "invalid_value" + ) + ) + continue + if not isinstance(spec, tuple): + continue + for position, item in enumerate(cast("tuple[object, ...]", spec)): + if _is_int(item) and cast(int, item) < 1: + problems.append( + ValidationProblem( + (*loc, position), + f"expected a positive chunk extent, got {item}", + "invalid_value", + ) + ) + elif isinstance(item, tuple): + size, count = cast("tuple[int, int]", item) + if size < 1 or count < 1: + problems.append( + ValidationProblem( + (*loc, position), + f"expected a positive [size, count] pair, got {item!r}", + "invalid_value", + ) + ) + return tuple(problems) + + +@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, requires=frozenset({"shape"})) +def tiles_the_array( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """One spec per dimension, and explicit specs must sum to that extent. + + A bare-integer dimension spec is uniform shorthand and imposes no sum + constraint; an explicit list of chunk sizes must tile its dimension + exactly. + """ + shape = document["shape"] + if not isinstance(shape, (list, tuple)): + return () + extents = cast("tuple[object, ...]", shape) + chunk_shapes = cast("tuple[object, ...]", configuration["chunk_shapes"]) + if len(chunk_shapes) != len(extents): + return ( + ValidationProblem( + ("chunk_shapes",), + f"chunk_shapes has {len(chunk_shapes)} entries but shape has " + f"{len(extents)} dimensions", + "invalid_value", + ), + ) + problems: list[ValidationProblem] = [] + for dim, (spec, extent) in enumerate(zip(chunk_shapes, extents, strict=True)): + if not _is_int(extent) or _is_int(spec) or not isinstance(spec, tuple): + continue + total = _expanded_extent(cast("tuple[object, ...]", spec)) + if total is not None and total < cast("int", extent): + problems.append( + ValidationProblem( + ("chunk_shapes", dim), + f"chunk sizes sum to {total} but must cover shape[{dim}] extent {extent}", + "invalid_value", + ) + ) + return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py new file mode 100644 index 0000000000..0a19b10133 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py @@ -0,0 +1,60 @@ +"""Composition rules for the `regular` chunk grid.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import CHUNK_GRID +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArraySpec + +_ARRAY_V3 = "zarr_v3_array" + + +@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME) +def chunk_extents_are_positive( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """Every chunk extent must be at least one element. + + A zero extent makes the chunk index `floor(i / 0)` undefined; a + negative one is meaningless. The shape validator enforces that the + entries are integers, so this rule judges only their values. + """ + chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(chunk_shape) + if extent < 1 + ) + + +@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, requires=frozenset({"shape"})) +def chunks_every_dimension( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """A regular grid must chunk every array dimension.""" + shape = document["shape"] + if not isinstance(shape, (list, tuple)): + return () + chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) + if len(chunk_shape) == len(cast("tuple[object, ...]", shape)): + return () + return ( + ValidationProblem( + ("chunk_shape",), + f"chunk_shape has {len(chunk_shape)} entries but shape has " + f"{len(cast('tuple[object, ...]', shape))} dimensions", + "invalid_value", + ), + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py new file mode 100644 index 0000000000..ccc74c1c01 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py @@ -0,0 +1,150 @@ +"""Composition rules for the `sharding_indexed` codec. + +Sharding is the one entity whose configuration contains whole pipelines +and its own geometry, so its rules recurse: the inner `codecs` and +`index_codecs` are judged by the same pipeline checks that judge the +document's top-level `codecs`, at every nesting depth. + +Every geometry judgment here is against the *incoming* array spec — the +array as transformed by every codec before this one — never against the +document's chunk grid directly. A `transpose` in front of a shard changes +which extents the shard has to divide, and reading the grid instead gave +wrong verdicts in both directions: it accepted an inner chunk that did +not divide the transposed shape and rejected one that did. + +The inner pipeline receives the inner chunk as its incoming spec (with +the incoming data type carried through), so a transpose or nested shard +inside it is judged against the inner chunk, recursively — each sharding +level encloses the next. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems +from zarr_metadata.rules._registry import entity_rule, run_chain_rules +from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec +from zarr_metadata.v3._extension_points import CODECS +from zarr_metadata.v3._shape import entity_name +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME +from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC_NAME +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + +_ARRAY_V3 = "zarr_v3_array" +_VARIABLE_SIZE_CODECS = frozenset( + {BLOSC_CODEC_NAME, GZIP_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME, ZSTD_CODEC_NAME} +) + + +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +def inner_chunk_extents_are_positive( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(chunk_shape) + if extent < 1 + ) + + +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +def inner_chunks_tile_the_incoming_array( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """The inner chunk must rank-match and evenly divide the array it receives. + + Declines when the incoming shape is unknown — an unclassified codec + upstream, or a non-regular grid at the top level — rather than + guessing from the document. + """ + if incoming.shape is None: + return () + outer = incoming.shape + inner = cast("tuple[int, ...]", configuration["chunk_shape"]) + if len(inner) != len(outer): + return ( + ValidationProblem( + ("chunk_shape",), + f"chunk_shape has {len(inner)} entries but the incoming array has " + f"{len(outer)} dimensions", + "invalid_value", + ), + ) + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"inner chunk extent {inner_extent} does not evenly divide the " + f"incoming extent {outer_extent}", + "invalid_value", + ) + for position, (outer_extent, inner_extent) in enumerate(zip(outer, inner, strict=True)) + if inner_extent >= 1 and outer_extent % inner_extent != 0 + ) + + +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +def inner_pipelines_are_pipelines( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """`codecs` and `index_codecs` obey the pipeline rules, recursively. + + Both get the ordering and shape judgments the top-level pipeline gets, + plus the entity rules of whatever codecs appear inside. The inner + `codecs` chain starts from the inner chunk with the incoming data + type; a nested shard or transpose inside it is therefore judged + against the inner chunk, and its own transitions carry on from there. + The `index_codecs` chain encodes the shard index, a `uint64` array + whose shape this package does not compute. + """ + inner_shape = configuration["chunk_shape"] + if not isinstance(inner_shape, tuple) or not all( + isinstance(v, int) and not isinstance(v, bool) and v >= 1 + for v in cast("tuple[object, ...]", inner_shape) + ): + inner_start = NOTHING_KNOWN + else: + # The inner pipeline encodes the inner chunk: same type as arrived + # here, shape of one inner chunk. + inner_start = incoming.with_shape(cast("tuple[int, ...]", inner_shape)) + problems: list[ValidationProblem] = [] + for key in ("codecs", "index_codecs"): + entries = configuration[key] + if not isinstance(entries, (list, tuple)): + continue + sequence = cast("tuple[object, ...]", entries) + problems.extend(pipeline_order_problems(sequence, (key,))) + problems.extend(shape_problems(sequence, (key,))) + # The index pipeline encodes the shard index, not the array: a + # uint64 array of offsets and lengths, so e.g. the bytes codec + # inside it still needs an endianness. + start = inner_start if key == "codecs" else ArraySpec(None, "uint64") + problems.extend(run_chain_rules(CODECS, sequence, document, (key,), start)) + return tuple(problems) + + +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +def index_codecs_have_fixed_encoded_size( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """The shard index must have an encoded size derivable from metadata.""" + entries = cast("tuple[object, ...]", configuration["index_codecs"]) + return tuple( + ValidationProblem( + ("index_codecs", index), + f"{name!r} produces variable-size output; index_codecs must be fixed-size", + "invalid_value", + ) + for index, entry in enumerate(entries) + if (name := entity_name(entry)) in _VARIABLE_SIZE_CODECS + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py new file mode 100644 index 0000000000..f0739735d1 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py @@ -0,0 +1,164 @@ +"""Composition rules for the `struct` data type. + +`StructField`'s own docstring promises field names are unique within a +struct and non-empty. Neither is expressible in a TypedDict, so both are +composition judgments and live here. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._engine import as_string_mapping +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import DATA_TYPE +from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME + +if TYPE_CHECKING: + from zarr_metadata.rules._spec import ArraySpec + +_ARRAY_V3 = "zarr_v3_array" + +_FIXED_SIZE_NAMES = frozenset( + { + "bool", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "float16", + "float32", + "float64", + "complex64", + "complex128", + "numpy.datetime64", + "numpy.timedelta64", + } +) +_VARIABLE_SIZE_NAMES = frozenset({"bytes", "string"}) + + +def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str], ...]: + """`(index, name)` for each field with a string name, else nothing. + + Anything the shape validator would reject is skipped: it owns that + complaint, and judging names inside a malformed field list is noise. + """ + fields = configuration.get("fields") + if not isinstance(fields, tuple): + return () + named: list[tuple[int, str]] = [] + for index, field in enumerate(cast("tuple[object, ...]", fields)): + if not isinstance(field, Mapping): + continue + name = cast("Mapping[object, object]", field).get("name") + if isinstance(name, str): + named.append((index, name)) + return tuple(named) + + +def _known_fixed_size(data_type: object) -> bool | None: + """Whether a known data type is fixed-size; None means unknown.""" + if isinstance(data_type, str): + name = data_type + envelope = None + else: + envelope = as_string_mapping(data_type) + raw_name = envelope.get("name") if envelope is not None else None + name = raw_name if isinstance(raw_name, str) else None + if name in _FIXED_SIZE_NAMES or ( + isinstance(name, str) and RAW_BYTES_NAME_PATTERN.fullmatch(name) + ): + return True + if name in _VARIABLE_SIZE_NAMES: + return False + if name != STRUCT_DATA_TYPE_NAME or envelope is None: + return None + nested_configuration = as_string_mapping(envelope.get("configuration")) + fields = nested_configuration.get("fields") if nested_configuration is not None else None + if not isinstance(fields, tuple): + return None + results: list[bool] = [] + for field in cast("tuple[object, ...]", fields): + field_mapping = as_string_mapping(field) + if field_mapping is None or "data_type" not in field_mapping: + return None + result = _known_fixed_size(field_mapping["data_type"]) + if result is None: + return None + results.append(result) + return all(results) + + +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +def fields_are_non_empty( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + fields = cast("tuple[object, ...]", configuration["fields"]) + if len(fields) != 0: + return () + return (ValidationProblem(("fields",), "expected at least one struct field", "invalid_value"),) + + +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +def field_data_types_are_fixed_size( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + fields = cast("tuple[object, ...]", configuration["fields"]) + problems: list[ValidationProblem] = [] + for index, field in enumerate(fields): + field_mapping = as_string_mapping(field) + if field_mapping is None or "data_type" not in field_mapping: + continue + if _known_fixed_size(field_mapping["data_type"]) is False: + problems.append( + ValidationProblem( + ("fields", index, "data_type"), + "struct fields must use fixed-size data types", + "invalid_value", + ) + ) + return tuple(problems) + + +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +def field_names_are_non_empty( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """A struct field must be addressable, so its name cannot be empty.""" + return tuple( + ValidationProblem( + ("fields", index, "name"), "expected a non-empty field name", "invalid_value" + ) + for index, name in _field_names(configuration) + if name == "" + ) + + +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +def field_names_are_unique( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """Duplicate field names make a fill value's per-field mapping ambiguous.""" + seen: dict[str, int] = {} + problems: list[ValidationProblem] = [] + for index, name in _field_names(configuration): + first = seen.get(name) + if first is None: + seen[name] = index + continue + problems.append( + ValidationProblem( + ("fields", index, "name"), + f"duplicate field name {name!r}, already used by field {first}", + "invalid_value", + ) + ) + return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py new file mode 100644 index 0000000000..61d98f0b4b --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py @@ -0,0 +1,78 @@ +"""Composition rules and spec transition for the `transpose` codec.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.rules._spec import ArraySpec, spec_transition +from zarr_metadata.v3._extension_points import CODECS +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + +_ARRAY_V3 = "zarr_v3_array" + + +@spec_transition(TRANSPOSE_CODEC_NAME) +def permute_shape(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: + """The outgoing shape is the incoming shape permuted by `order`. + + Declines (shape None) when the order is not a permutation of the + incoming rank: the rules below report that, and any shape derived + from a bad order would be a guess. + """ + order = cast("tuple[int, ...]", configuration["order"]) + shape = incoming.shape + if shape is None or sorted(order) != list(range(len(shape))): + return incoming.with_shape(None) + return incoming.with_shape(tuple(shape[axis] for axis in order)) + + +@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME) +def order_is_a_permutation( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """`order` must be a permutation of its own indices. + + Checked without reference to the incoming shape, so it holds even + when propagation has stopped upstream. + """ + order = cast("tuple[int, ...]", configuration["order"]) + if sorted(order) == list(range(len(order))): + return () + return ( + ValidationProblem( + ("order",), + f"expected a permutation of 0..{len(order) - 1}, got {order!r}", + "invalid_value", + ), + ) + + +@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME) +def order_matches_incoming_rank( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """A transpose permutes the array it receives, so ranks must agree. + + Judged against the *incoming* spec, not the document's `shape`: inside + a shard the incoming array is the inner chunk, and after another + transpose it is that transpose's output. Declines when the incoming + shape is unknown. + """ + if incoming.shape is None: + return () + order = cast("tuple[int, ...]", configuration["order"]) + if len(order) == len(incoming.shape): + return () + return ( + ValidationProblem( + ("order",), + f"order has {len(order)} entries but the incoming array has " + f"{len(incoming.shape)} dimensions", + "invalid_value", + ), + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py new file mode 100644 index 0000000000..34a05deb50 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py @@ -0,0 +1,119 @@ +"""Codec-pipeline judgments, shared by the array rules and by sharding. + +A sharding codec's `codecs` and `index_codecs` are pipelines exactly like +the document's top-level `codecs`, so the ordering and shape checks live +here rather than in either caller: sharding recurses into them at every +nesting depth, and the top-level array rules apply them at depth zero. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._engine import as_string_mapping, prefixed +from zarr_metadata.v3._shape import entity_name, validate_known_codec_metadata +from zarr_metadata.v3.codec.kind import codec_kind_of_name + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr_metadata.v3.codec.kind import CodecKind + +_KIND_RANK: Final = {"array_array": 0, "array_bytes": 1, "bytes_bytes": 2} + + +def codec_kind(codec: object) -> CodecKind | None: + """The pipeline kind of `codec`, classified by name alone. + + Spelling-insensitive on purpose: a known codec in an invalid spelling + still ranks as its kind, so two spellings of the same pipeline always + get the same ordering verdict and a misspelled known codec is never + mistaken for an unknown extension (which would suppress the + exactly-one-`array->bytes` count). + """ + name = entity_name(codec) + if name is None: + return None + return codec_kind_of_name(name) + + +def _codec_label(codec: object) -> str: + if isinstance(codec, str): + return repr(codec) + mapping = as_string_mapping(codec) + if mapping is not None: + return repr(mapping.get("name")) + return repr(codec) + + +def pipeline_order_problems( + entries: Sequence[object], loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + """The spec pipeline shape: `array->array`* `array->bytes` `bytes->bytes`*. + + Codecs of genuinely unknown name are skipped: they impose no ordering + constraint, and their presence makes the exactly-one-`array->bytes` + count inconclusive (an unknown codec might be the pipeline's + `array->bytes` stage), so that check only fires when every codec is + classified. + """ + + problems: list[ValidationProblem] = [] + kinds = [codec_kind(codec) for codec in entries] + max_rank_seen = -1 + array_bytes_count = 0 + for index, (codec, kind) in enumerate(zip(entries, kinds, strict=True)): + if kind is None: + continue + rank = _KIND_RANK[kind] + if rank < max_rank_seen: + problems.append( + ValidationProblem( + (*loc, index), + f"{kind.replace('_', '->')} codec {_codec_label(codec)} may not " + "follow a later-stage codec in the pipeline", + "invalid_value", + ) + ) + max_rank_seen = max(max_rank_seen, rank) + if kind == "array_bytes": + array_bytes_count += 1 + if array_bytes_count > 1: + problems.append( + ValidationProblem( + (*loc, index), + f"extra array->bytes codec {_codec_label(codec)}: a pipeline " + "has exactly one", + "invalid_value", + ) + ) + if array_bytes_count == 0 and all(kind is not None for kind in kinds): + problems.append( + ValidationProblem(loc, "codec pipeline has no array->bytes codec", "invalid_value") + ) + return tuple(problems) + + +def shape_problems( + entries: Sequence[object], loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + """Shape problems for every known-name codec entry in `entries`. + + Unknown names pass untouched (extension openness); entries without an + interpretable name decline in favor of the structural validator. + """ + problems: list[ValidationProblem] = [] + for index, codec in enumerate(entries): + found = validate_known_codec_metadata(codec) + # None is "not a known codec" (unjudged); () is "known and valid". + if found is not None: + problems.extend(prefixed((*loc, index), found)) + return tuple(problems) + + +__all__ = [ + "codec_kind", + "pipeline_order_problems", + "shape_problems", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py new file mode 100644 index 0000000000..042d3542bb --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -0,0 +1,319 @@ +"""Register rules by document type and extension entity. + +`@document_rule` and `@entity_rule` register checks where they are +defined, so a rule cannot be written without joining the set it belongs +to. Both reject dependencies absent from the document type. Entity rules +are keyed by `(field, canonical_name)` and require a corresponding shape +validator. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, cast + +from zarr_metadata.rules._engine import Rule, as_string_mapping, prefixed +from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec, initial_spec, propagate +from zarr_metadata.v3._extension_points import CHUNK_GRID, ExtensionPointField, canonical_name +from zarr_metadata.v3._shape import ( + blocking_problems, + entity_name, + modelled_entities, + validate_known_entity_metadata, +) + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ValidationProblem + +EntityCheck = Callable[ + [Mapping[str, object], Mapping[str, object], "ArraySpec"], + "tuple[ValidationProblem, ...]", +] +"""An entity rule's check: `(configuration, document, incoming)` in, problems out. + +`incoming` is the `ArraySpec` the entity receives — for a codec, the +array as transformed by every codec before it in the chain. Fields this +package cannot determine are `None`; a caller with no chain context +passes `NOTHING_KNOWN`. Rules that need a field test it `is None` and +decline; rules that do not simply ignore the spec. + +Problems carry locations relative to the entity's `configuration`; the +dispatcher re-bases them onto the entity's position in the document. +""" + + +@dataclass(frozen=True, slots=True) +class EntityRule: + """One composition check for a named extension entity. + + Identified by `(field, entity)`, never by name alone: names are + unique only within an extension point, and `bytes` is both a core + codec and a registered extension data type. Keying by name would + make a rule written for one fire on the other. + + `requires` are *document* keys the check reads beyond the entity + itself (e.g. `shape`), gating the rule exactly as `Rule.requires` + does. + """ + + field: str + entity: str + requires: frozenset[str] + check: EntityCheck + + +_DOCUMENT_RULES: Final[dict[str, list[Rule]]] = defaultdict(list) +_ENTITY_RULES: Final[dict[tuple[str, str], list[EntityRule]]] = defaultdict(list) +_DOCUMENT_KEYS: Final[dict[str, frozenset[str]]] = {} +_DISPATCHED_FIELDS: Final[set[str]] = set() + + +def register_document_type( + document_type: str, + standard_keys: frozenset[str], + extension_keys: frozenset[str] = frozenset(), +) -> None: + """Declare a document type's known keys, so `requires` can be checked. + + `extension_keys` names keys that are not part of the document's + TypedDict but that this package nonetheless recognizes — the v3 + `consolidated_metadata` convention is the only one today. Requiring + them to be declared here rather than exempting unknown keys wholesale + keeps the typo check meaningful. + """ + _DOCUMENT_KEYS[document_type] = standard_keys | extension_keys + + +def _validate_requires(document_type: str, requires: frozenset[str], what: str) -> None: + known = _DOCUMENT_KEYS.get(document_type) + if known is None: + msg = f"unknown document type {document_type!r} registering {what}" + raise LookupError(msg) + unknown = requires - known + if len(unknown) != 0: + msg = ( + f"{what} requires {sorted(unknown)}, which {document_type} documents " + f"do not have; such a rule could never fire" + ) + raise ValueError(msg) + + +def document_rule( + document_type: str, requires: frozenset[str] +) -> Callable[[Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]], Rule]: + """Register a whole-document rule, returning the `Rule` it becomes. + + The decorated function is replaced by its `Rule`, so a rule cannot be + defined without being registered, and referencing one by name yields + the registered object rather than a copy. + """ + + def decorate( + check: Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]], + ) -> Rule: + _validate_requires(document_type, requires, f"rule {check.__name__!r}") + rule = Rule(requires=requires, check=check) + _DOCUMENT_RULES[document_type].append(rule) + return rule + + return decorate + + +def entity_rule( + document_type: str, + field: ExtensionPointField, + entity: str, + requires: frozenset[str] = frozenset(), +) -> Callable[[EntityCheck], EntityRule]: + """Register a rule about one named entity within `document_type`. + + The entity must already be shape-modelled in `zarr_metadata.v3._shape`: + entity rules read configuration members by name, so they only run once + the shape validator vouches those members exist and are typed. A rule + registered for an unmodelled name would silently never fire, so that + is refused here rather than discovered as a missing check later. + """ + + def decorate(check: EntityCheck) -> EntityRule: + _validate_requires(document_type, requires, f"entity rule {check.__name__!r}") + canonical_entity = canonical_name(field, entity) + if (field, canonical_entity) not in modelled_entities(): + msg = ( + f"entity rule {check.__name__!r} targets {entity!r}, which has no shape " + f"validator in zarr_metadata.v3._shape; such a rule could never fire" + ) + raise ValueError(msg) + rule = EntityRule(field=field, entity=entity, requires=requires, check=check) + _ENTITY_RULES[field, canonical_entity].append(rule) + return rule + + return decorate + + +def document_rules(document_type: str) -> tuple[Rule, ...]: + """Every rule registered for `document_type`, in definition order.""" + return tuple(_DOCUMENT_RULES[document_type]) + + +def dispatched_fields() -> frozenset[str]: + """Extension points that have a dispatcher, so their rules can run. + + An entity rule registered at a field with no dispatcher is accepted and + then never fires — the silent-pass failure this module exists to + prevent. Checking coverage at registration would depend on import + order, so `tests/rules/test_registry.py` asserts it instead. + """ + return frozenset(_DISPATCHED_FIELDS) + + +def registered_entities() -> frozenset[tuple[str, str]]: + """Every `(field, canonical name)` that has at least one registered rule.""" + return frozenset(_ENTITY_RULES) + + +def run_entity_rules( + field: ExtensionPointField, + value: object, + document: Mapping[str, object], + loc: tuple[str | int, ...], + incoming: ArraySpec = NOTHING_KNOWN, +) -> tuple[ValidationProblem, ...]: + """Run the rules registered for whatever entity `value` names. + + Declines silently when `value` names nothing known, when its shape is + broken in a way that makes its configuration uninterpretable (the + shape rule owns that complaint), or when a rule's required document + keys are absent. An `unknown_key` never declines — see + `zarr_metadata.v3._shape.blocking_problems`. + """ + name = entity_name(value) + if name is None: + return () + rules = _ENTITY_RULES.get((field, canonical_name(field, name))) + if rules is None or len(rules) == 0: + return () + # Entity rules read configuration members by name, so they may only run + # once the shape validator vouches those members exist and are typed. + configuration = entity_configuration(field, value) + if configuration is None: + return () + problems: list[ValidationProblem] = [] + for rule in rules: + if not rule.requires <= document.keys(): + continue + problems.extend( + prefixed((*loc, "configuration"), rule.check(configuration, document, incoming)) + ) + return tuple(problems) + + +def entity_configuration(field: ExtensionPointField, value: object) -> Mapping[str, object] | None: + """`value`'s configuration if its modelled fields are usable, else None. + + Shared by the dispatchers and by rules that reach across entities + (sharding's nested pipelines). `unknown_key` problems do not make an + entity unusable; anything else does. + """ + verdict = validate_known_entity_metadata(field, value) + if verdict is None or len(blocking_problems(verdict)) != 0: + return None + mapping = as_string_mapping(value) + if mapping is None: + # Bare-string metadata is the canonical spelling for entities whose + # configuration is optional. Rules still need a real mapping to run + # against, especially when they judge a missing optional member. + return {} if isinstance(value, str) else None + if "configuration" not in mapping: + return {} + return as_string_mapping(mapping["configuration"]) + + +def dispatch_field( + field: ExtensionPointField, +) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: + """A check that runs entity rules for the entity in `document[field]`.""" + + def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + return run_entity_rules(field, document[field], document, (field,)) + + _DISPATCHED_FIELDS.add(field) + check.__name__ = f"_dispatch_{field}_entity_rules" + return check + + +def dispatch_field_sequence( + field: ExtensionPointField, +) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: + """A check that runs entity rules for every entity in `document[field]`.""" + + def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + entries = document[field] + if not isinstance(entries, (list, tuple)): + return () + sequence = cast("tuple[object, ...]", entries) + return run_chain_rules(field, sequence, document, (field,), chain_initial_spec(document)) + + _DISPATCHED_FIELDS.add(field) + check.__name__ = f"_dispatch_{field}_entity_rules" + return check + + +def run_chain_rules( + field: ExtensionPointField, + codecs: Sequence[object], + document: Mapping[str, object], + loc: tuple[str | int, ...], + initial: ArraySpec, +) -> tuple[ValidationProblem, ...]: + """Run entity rules over a codec chain, propagating the array spec. + + Each codec's rules receive the spec that codec actually receives — + the array as transformed by everything before it. Shared by the + top-level `codecs` dispatcher and by sharding, whose inner pipelines + are chains that start from the inner chunk. + """ + problems: list[ValidationProblem] = [] + for index, entry, incoming in propagate( + codecs, initial, lambda codec: entity_configuration(field, codec) + ): + problems.extend(run_entity_rules(field, entry, document, (*loc, index), incoming)) + return tuple(problems) + + +def chain_initial_spec(document: Mapping[str, object]) -> ArraySpec: + """The spec entering a document's top-level codec chain. + + The array a chunk pipeline encodes is one chunk: shape from a regular + grid this package can read (None otherwise), data type from the + document. Non-positive chunk extents yield None for the shape — the + grid's own values rule owns that complaint, and geometry against a + zero extent is noise on top of it. + """ + from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME + + grid = document.get("chunk_grid") + chunk_shape: tuple[int, ...] | None = None + if entity_name(grid) == REGULAR_CHUNK_GRID_NAME: + configuration = entity_configuration(CHUNK_GRID, grid) + extents = configuration.get("chunk_shape") if configuration is not None else None + if isinstance(extents, tuple): + values = cast("tuple[object, ...]", extents) + if all(isinstance(v, int) and not isinstance(v, bool) and v >= 1 for v in values): + chunk_shape = cast("tuple[int, ...]", values) + return initial_spec(document, chunk_shape) + + +__all__ = [ + "EntityCheck", + "EntityRule", + "chain_initial_spec", + "dispatched_fields", + "document_rule", + "document_rules", + "entity_rule", + "registered_entities", + "run_chain_rules", + "run_entity_rules", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_result.py b/packages/zarr-metadata/src/zarr_metadata/rules/_result.py new file mode 100644 index 0000000000..4b099c6947 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_result.py @@ -0,0 +1,101 @@ +"""Tagged validation results. + +`check_*` returns `Valid[T] | Invalid`. Testing the literal `valid` +field narrows to either the normalized document or a nonempty problem +tuple. Use `validate_*` to collect problems and `parse_*` to raise on +invalid input. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Generic, Literal, TypeAlias, TypeVar, cast + +from zarr_metadata.model._validation import ValidationProblem, arrays_to_tuples +from zarr_metadata.rules._documents import ( + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) +from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON # noqa: TC001 +from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON # noqa: TC001 +from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON # noqa: TC001 +from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON # noqa: TC001 + +DocumentT = TypeVar("DocumentT") + + +@dataclass(frozen=True, slots=True) +class Valid(Generic[DocumentT]): + """A document that passed structural and composition validation.""" + + document: DocumentT + valid: Literal[True] = True + + +@dataclass(frozen=True, slots=True) +class Invalid: + """Every reason a document failed validation. + + `problems` is never empty: an empty report is a `Valid`. + """ + + problems: tuple[ValidationProblem, ...] + valid: Literal[False] = False + + def __post_init__(self) -> None: + if len(self.problems) == 0: + msg = "Invalid requires at least one validation problem" + raise ValueError(msg) + + +ValidationResult: TypeAlias = Valid[DocumentT] | Invalid +"""Either a validated document or the problems that disqualified it.""" + + +def check_array_metadata_v3(value: object) -> ValidationResult[ZarrV3ArrayMetadataJSON]: + """`value` as a valid v3 array document, or the problems disqualifying it. + + A `Valid` carries the normalized document (JSON arrays as tuples), + exactly as `parse_array_metadata_v3` returns it. + """ + problems = validate_array_metadata_v3(value) + if len(problems) != 0: + return Invalid(problems) + return Valid(cast("ZarrV3ArrayMetadataJSON", arrays_to_tuples(value))) + + +def check_array_metadata_v2(value: object) -> ValidationResult[ZarrV2ArrayMetadataJSON]: + """`value` as a valid v2 array document, or the problems disqualifying it.""" + problems = validate_array_metadata_v2(value) + if len(problems) != 0: + return Invalid(problems) + return Valid(cast("ZarrV2ArrayMetadataJSON", arrays_to_tuples(value))) + + +def check_group_metadata_v3(value: object) -> ValidationResult[ZarrV3GroupMetadataJSON]: + """`value` as a valid v3 group document, or the problems disqualifying it.""" + problems = validate_group_metadata_v3(value) + if len(problems) != 0: + return Invalid(problems) + return Valid(cast("ZarrV3GroupMetadataJSON", arrays_to_tuples(value))) + + +def check_group_metadata_v2(value: object) -> ValidationResult[ZarrV2GroupMetadataJSON]: + """`value` as a valid v2 group document, or the problems disqualifying it.""" + problems = validate_group_metadata_v2(value) + if len(problems) != 0: + return Invalid(problems) + return Valid(cast("ZarrV2GroupMetadataJSON", arrays_to_tuples(value))) + + +__all__ = [ + "Invalid", + "Valid", + "ValidationResult", + "check_array_metadata_v2", + "check_array_metadata_v3", + "check_group_metadata_v2", + "check_group_metadata_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py new file mode 100644 index 0000000000..88459c26b4 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py @@ -0,0 +1,156 @@ +"""Array specifications and how a codec chain transforms them. + +Each array->array codec transforms the array it receives, so a codec's +configuration must be judged against the array that *reaches* it, not +against the document's top-level fields: `transpose` permutes the shape, +`cast_value` changes the data type, and a `sharding_indexed` codec that +follows either one sees the transformed array. + +`ArraySpec` is the array a codec receives; `propagate` walks a chain +handing each codec its incoming spec. A field is `None` once this package +can no longer determine it. An unknown codec might change anything, so +every codec after one receives `NOTHING_KNOWN` and rules that need a +field decline rather than guess. Shape stops at the array->bytes +boundary; the data type carries through. + +Transitions are registered per array->array codec, next to that codec's +rules, via `spec_transition`. A modelled codec with no transition is +treated as unknown, so a forgotten transition fails closed. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Final + +from zarr_metadata.v3._extension_points import CODECS, canonical_name +from zarr_metadata.v3._shape import entity_name +from zarr_metadata.v3.codec.kind import codec_kind_of_name + +if TYPE_CHECKING: + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + + +@dataclass(frozen=True, slots=True) +class ArraySpec: + """The array a codec receives; a field is `None` when undetermined. + + `data_type` is the metadata-field value verbatim (a bare name or a + name/configuration object) because rules compare it by name. + """ + + shape: tuple[int, ...] | None + data_type: ZarrV3MetadataFieldJSON | None + + def with_shape(self, shape: tuple[int, ...] | None) -> ArraySpec: + return replace(self, shape=shape) + + def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArraySpec: + return replace(self, data_type=data_type) + + +NOTHING_KNOWN: Final = ArraySpec(None, None) +"""The spec past a point where nothing about the array can be determined. + +Compare by equality: a spec can arrive here field by field and is then +equal to this constant without being it. +""" + + +SpecTransition = Callable[[Mapping[str, object], ArraySpec], ArraySpec] +"""How one codec transforms the spec it receives. + +Takes the codec's (shape-valid) configuration and the incoming spec, and +returns the outgoing one. A transition must never raise on the values the +shape validator admits; a field it cannot determine becomes `None`. +""" + +_TRANSITIONS: Final[dict[str, SpecTransition]] = {} + + +def spec_transition(codec: str) -> Callable[[SpecTransition], SpecTransition]: + """Register how `codec` transforms an incoming `ArraySpec`. + + Only array->array codecs need one: array->bytes and bytes->bytes + codecs end shape propagation by construction, so registering a + transition for one is refused. + """ + kind = codec_kind_of_name(codec) + if kind != "array_array": + msg = ( + f"spec transition registered for {codec!r}, which is " + f"{kind or 'unknown'} rather than array_array; only array->array " + "codecs transform the array spec" + ) + raise ValueError(msg) + + def decorate(transition: SpecTransition) -> SpecTransition: + _TRANSITIONS[canonical_name(CODECS, codec)] = transition + return transition + + return decorate + + +def transitions_registered() -> frozenset[str]: + """Every codec name with a registered spec transition.""" + return frozenset(_TRANSITIONS) + + +def propagate( + codecs: Sequence[object], + initial: ArraySpec, + configuration_of: Callable[[object], Mapping[str, object] | None], +) -> Iterator[tuple[int, object, ArraySpec]]: + """Yield `(index, codec, incoming_spec)` for each codec in the chain. + + `incoming_spec` is `NOTHING_KNOWN` once propagation has stopped: after + an unknown codec, after a known codec whose configuration is not + shape-valid, or after a codec this package has no transition for. + `configuration_of` resolves a codec entry to its usable configuration + (`entity_configuration` in practice; injected to keep this module free + of the registry). + """ + spec = initial + for index, codec in enumerate(codecs): + yield index, codec, spec + if spec == NOTHING_KNOWN: + continue + name = entity_name(codec) + kind = codec_kind_of_name(name) if name is not None else None + if kind is None: + spec = NOTHING_KNOWN + elif kind == "array_array": + transition = _TRANSITIONS.get(canonical_name(CODECS, name or "")) + configuration = configuration_of(codec) + if transition is None or configuration is None: + spec = NOTHING_KNOWN + else: + spec = transition(configuration, spec) + else: + # array->bytes: the array is gone; bytes->bytes: never had one. + spec = spec.with_shape(None) + + +def initial_spec(document: Mapping[str, object], chunk_shape: tuple[int, ...] | None) -> ArraySpec: + """The spec entering a document's top-level codec chain. + + The array a chunk pipeline encodes is one chunk, so the incoming shape + is the chunk grid's chunk shape (`None` if the grid is not a regular + grid this package can read). The data type is the document's own. + """ + data_type = document.get("data_type") + if not isinstance(data_type, (str, Mapping)): + data_type = None + return ArraySpec(chunk_shape, data_type) # type: ignore[arg-type] + + +__all__ = [ + "NOTHING_KNOWN", + "ArraySpec", + "SpecTransition", + "initial_spec", + "propagate", + "spec_transition", + "transitions_registered", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py new file mode 100644 index 0000000000..cd4493dc59 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py @@ -0,0 +1,54 @@ +"""Composition rules for v2 array metadata documents. + +The v2 rule set is deliberately small today: the one cross-field +constraint the package interprets is that `chunks` and `shape` agree on +dimensionality. Fill-value/dtype consistency for v2 (NumPy dtype strings, +base64 fills for bytes dtypes) is a known follow-up, tracked in the +package docs. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V2, + ValidationProblem, +) +from zarr_metadata.rules._engine import Rule, as_sequence +from zarr_metadata.rules._registry import document_rule, document_rules, register_document_type + +if TYPE_CHECKING: + from collections.abc import Mapping + + +ZARR_V2_ARRAY = "zarr_v2_array" +"""Document-type key under which this module's rules are registered.""" + +register_document_type(ZARR_V2_ARRAY, ARRAY_METADATA_STANDARD_KEYS_V2) + + +@document_rule(ZARR_V2_ARRAY, frozenset({"shape", "chunks"})) +def check_chunks_match_shape(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """`chunks` must have one entry per dimension of `shape`.""" + shape = as_sequence(document["shape"]) + chunks = as_sequence(document["chunks"]) + if shape is None or chunks is None or len(shape) == len(chunks): + return () + return ( + ValidationProblem( + ("chunks",), + "expected the same number of dimensions as shape", + "invalid_value", + ), + ) + + +ZARR_V2_ARRAY_RULES: Final[tuple[Rule, ...]] = document_rules(ZARR_V2_ARRAY) +"""The composition rule set for v2 array metadata documents.""" + + +__all__ = [ + "ZARR_V2_ARRAY", + "ZARR_V2_ARRAY_RULES", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py new file mode 100644 index 0000000000..1391707ddb --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py @@ -0,0 +1,428 @@ +"""Composition rules for v3 array metadata documents. + +Whole-document rules live here: judgments that read several top-level +fields, or that apply to a field regardless of which extension occupies +it. Rules about a *particular* codec or chunk grid live with that entity +in `zarr_metadata.rules._entities`, registered by name — so adding a +third chunk grid or a new codec adds a module there and changes nothing +in this one. The `codecs` and `chunk_grid` dispatchers below are generic: +they run whatever rules are registered for the name they find. + +Extension openness: rules never reject what they cannot interpret. An +unknown data type name accepts any fill value here (its own validator is +whoever understands it), an unknown codec has unknown kind, and unknown +entities pass through untouched. Openness is for genuinely unknown names +only: a codec or chunk-grid name this package defines is held to its full +canonical shape (via `zarr_metadata.v3._shape`), and a known codec ranks +as its pipeline kind in every spelling — otherwise a misspelled known +name would masquerade as an unknown extension and silently escape both +the shape and the ordering checks. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final, cast + +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + ValidationProblem, +) +from zarr_metadata.rules._engine import Rule, as_sequence, as_string_mapping, prefixed +from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems +from zarr_metadata.rules._registry import ( + dispatch_field, + dispatch_field_sequence, + document_rule, + document_rules, + register_document_type, +) +from zarr_metadata.v3._extension_points import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, + ExtensionPointField, +) +from zarr_metadata.v3._shape import ( + validate_known_chunk_grid_metadata, + validate_known_entity_metadata, +) +from zarr_metadata.v3.data_type.bytes import base64_bytes +from zarr_metadata.v3.data_type.float16 import hex_float16 +from zarr_metadata.v3.data_type.float32 import hex_float32 +from zarr_metadata.v3.data_type.float64 import hex_float64 +from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN, raw_bytes_dtype_name + +if TYPE_CHECKING: + from collections.abc import Callable + + +# --------------------------------------------------------------------------- +# fill_value vs. data_type +# --------------------------------------------------------------------------- + +_INT_RANGES: Final[dict[str, tuple[int, int]]] = { + "int8": (-(2**7), 2**7 - 1), + "int16": (-(2**15), 2**15 - 1), + "int32": (-(2**31), 2**31 - 1), + "int64": (-(2**63), 2**63 - 1), + "uint8": (0, 2**8 - 1), + "uint16": (0, 2**16 - 1), + "uint32": (0, 2**32 - 1), + "uint64": (0, 2**64 - 1), +} + +_FLOAT_HEX_VALIDATORS: Final[dict[str, Callable[[str], object]]] = { + "float16": hex_float16, + "float32": hex_float32, + "float64": hex_float64, +} + +_COMPLEX_COMPONENT_TYPES: Final[dict[str, str]] = { + "complex64": "float32", + "complex128": "float64", +} + +_FLOAT_SPECIALS: Final = frozenset({"NaN", "Infinity", "-Infinity"}) + + +def _is_int(value: object) -> bool: + # bool is an int subtype but is never a valid integer fill value. + return isinstance(value, int) and not isinstance(value, bool) + + +def _check_float_fill(value: object, dtype_name: str) -> str | None: + if _is_int(value) or isinstance(value, float): + return None + if isinstance(value, str): + if value in _FLOAT_SPECIALS: + return None + try: + _FLOAT_HEX_VALIDATORS[dtype_name](value) + except ValueError: + return ( + f"expected a number, one of 'NaN'/'Infinity'/'-Infinity', or a " + f"{dtype_name} hex string, got {value!r}" + ) + return None + return f"expected a number or string, got {value!r}" + + +def _check_byte_sequence(value: object, expected_len: int | None) -> str | None: + items = as_sequence(value) + if items is None: + return f"expected an array of byte values, got {value!r}" + if expected_len is not None and len(items) != expected_len: + return f"expected {expected_len} byte values, got {len(items)}" + for item in items: + if not _is_int(item) or not 0 <= cast(int, item) <= 255: + return f"expected integers in [0, 255], got {item!r}" + return None + + +def _check_fill_for_dtype(dtype_name: str, value: object) -> str | None: + """Why `value` is not a valid fill value for `dtype_name`, or None. + + Unknown data type names accept anything (extension openness). + """ + if dtype_name == "bool": + return None if isinstance(value, bool) else f"expected a boolean, got {value!r}" + if dtype_name in _INT_RANGES: + low, high = _INT_RANGES[dtype_name] + if not _is_int(value): + return f"expected an integer, got {value!r}" + if not low <= cast(int, value) <= high: + return f"expected an integer in [{low}, {high}], got {value!r}" + return None + if dtype_name in _FLOAT_HEX_VALIDATORS: + return _check_float_fill(value, dtype_name) + if dtype_name in _COMPLEX_COMPONENT_TYPES: + component = _COMPLEX_COMPONENT_TYPES[dtype_name] + pair = as_sequence(value) + if pair is None or len(pair) != 2: + return f"expected a [real, imag] pair, got {value!r}" + for part in pair: + reason = _check_float_fill(part, component) + if reason is not None: + return f"invalid component: {reason}" + return None + if dtype_name == "string": + return None if isinstance(value, str) else f"expected a string, got {value!r}" + if dtype_name == "bytes": + if isinstance(value, str): + try: + base64_bytes(value) + except ValueError: + return f"expected standard-alphabet base64, got {value!r}" + return None + return _check_byte_sequence(value, None) + if dtype_name in ("numpy.datetime64", "numpy.timedelta64"): + if value == "NaT": + return None + if not _is_int(value): + return f"expected a signed 64-bit integer or 'NaT', got {value!r}" + if not -(2**63) <= cast(int, value) <= 2**63 - 1: + return f"expected a signed 64-bit integer, got {value!r}" + return None + if dtype_name == "struct": + if isinstance(value, Mapping): + return None + return f"expected an object of per-field fill values, got {value!r}" + if RAW_BYTES_NAME_PATTERN.fullmatch(dtype_name) is not None: + try: + raw_bytes_dtype_name(dtype_name) + except ValueError: + return None # malformed r name: _check_data_type_spelling reports it + return _check_byte_sequence(value, int(dtype_name[1:]) // 8) + return None # unknown data type: its fill values are not ours to judge + + +def _dtype_name(data_type: object) -> str | None: + if isinstance(data_type, str): + return data_type + mapping = as_string_mapping(data_type) + if mapping is not None: + name = mapping.get("name") + if isinstance(name, str): + return name + return None # structurally invalid; the structural validator reports it + + +def _check_data_type_spelling(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Misspellings of data type families this package defines. + + An `r` name whose bit count is not a positive multiple of 8 is a + misspelling of the known raw-bytes family, not an unknown extension: + treating it as unknown would let the misspelling masquerade as an + extension and escape judgment entirely (the same anti-masquerade + reasoning as the codec spelling checks). Genuinely unknown names pass + untouched. + """ + name = _dtype_name(document["data_type"]) + if name is None or RAW_BYTES_NAME_PATTERN.fullmatch(name) is None: + return () + try: + raw_bytes_dtype_name(name) + except ValueError as error: + return (ValidationProblem(("data_type",), str(error), "invalid_value"),) + return () + + +def _check_fill_matches_dtype(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + data_type = document["data_type"] + dtype_name = _dtype_name(data_type) + if dtype_name is None: + return () + if dtype_name == "struct": + return _struct_fill_problems(data_type, document["fill_value"], ("fill_value",)) + reason = _check_fill_for_dtype(dtype_name, document["fill_value"]) + if reason is None: + return () + return ( + ValidationProblem( + ("fill_value",), + f"fill_value invalid for data_type {dtype_name!r}: {reason}", + "invalid_value", + ), + ) + + +def _struct_fill_problems( + data_type: object, fill_value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + """Validate a struct fill mapping against every field, recursively.""" + if not isinstance(fill_value, Mapping): + return ( + ValidationProblem( + loc, + f"fill_value invalid for data_type 'struct': expected an object of " + f"per-field fill values, got {fill_value!r}", + "invalid_value", + ), + ) + envelope = as_string_mapping(data_type) + configuration = ( + as_string_mapping(envelope.get("configuration")) if envelope is not None else None + ) + fields = configuration.get("fields") if configuration is not None else None + if not isinstance(fields, tuple): + return () # malformed data type: the structural validator owns it + + fill_mapping = cast("Mapping[object, object]", fill_value) + problems: list[ValidationProblem] = [] + field_names: set[str] = set() + for field in cast("tuple[object, ...]", fields): + field_mapping = as_string_mapping(field) + if field_mapping is None: + continue + name = field_mapping.get("name") + field_data_type = field_mapping.get("data_type") + if not isinstance(name, str) or field_data_type is None: + continue + field_names.add(name) + field_loc = (*loc, name) + if name not in fill_mapping: + problems.append( + ValidationProblem( + field_loc, f"missing fill value for struct field {name!r}", "missing_key" + ) + ) + continue + field_fill = fill_mapping[name] + nested_name = _dtype_name(field_data_type) + if nested_name == "struct": + problems.extend(_struct_fill_problems(field_data_type, field_fill, field_loc)) + continue + if nested_name is None: + continue + reason = _check_fill_for_dtype(nested_name, field_fill) + if reason is not None: + problems.append( + ValidationProblem( + field_loc, + f"fill value invalid for struct field {name!r} with data_type " + f"{nested_name!r}: {reason}", + "invalid_value", + ) + ) + problems.extend( + ValidationProblem((*loc, key), f"unknown struct fill field {key!r}", "unknown_key") + for key in sorted( + candidate + for candidate in fill_mapping.keys() - field_names + if isinstance(candidate, str) + ) + ) + return tuple(problems) + + +# --------------------------------------------------------------------------- +# whole-document rules +# --------------------------------------------------------------------------- + +ZARR_V3_ARRAY = "zarr_v3_array" +"""Document-type key under which this module's rules are registered.""" + +register_document_type(ZARR_V3_ARRAY, ARRAY_METADATA_STANDARD_KEYS_V3) + +_data_type_spelling = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( + _check_data_type_spelling +) +_fill_matches_dtype = document_rule(ZARR_V3_ARRAY, frozenset({"data_type", "fill_value"}))( + _check_fill_matches_dtype +) + + +def _known_entity_shape( + field: ExtensionPointField, +) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: + """A check that judges `document[field]` against `field`'s known shapes. + + One parameter, not two: the document field and the extension point are + the same thing, and taking them separately invited passing a codec + under the chunk-grid point. + """ + + def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + found = validate_known_entity_metadata(field, document[field]) + return () if found is None else prefixed((field,), found) + + return check + + +_data_type_shape = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( + _known_entity_shape(DATA_TYPE) +) +_chunk_key_encoding_shape = document_rule(ZARR_V3_ARRAY, frozenset({"chunk_key_encoding"}))( + _known_entity_shape(CHUNK_KEY_ENCODING) +) + + +@document_rule(ZARR_V3_ARRAY, frozenset({"codecs"})) +def check_codec_pipeline_order(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """The pipeline shape: `array->array`* `array->bytes` `bytes->bytes`*.""" + entries = as_sequence(document["codecs"]) + if entries is None: + return () + return pipeline_order_problems(entries, ("codecs",)) + + +@document_rule(ZARR_V3_ARRAY, frozenset({"codecs"})) +def check_codec_shapes(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Every known-name codec matches its canonical type.""" + entries = as_sequence(document["codecs"]) + if entries is None: + return () + return shape_problems(entries, ("codecs",)) + + +@document_rule(ZARR_V3_ARRAY, frozenset({"chunk_grid"})) +def check_chunk_grid_shape(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """A known-name chunk grid matches its canonical type.""" + found = validate_known_chunk_grid_metadata(document["chunk_grid"]) + # None is "not a known grid" (unjudged); () is "known and valid". + if found is None: + return () + return prefixed(("chunk_grid",), found) + + +@document_rule(ZARR_V3_ARRAY, frozenset({"shape", "dimension_names"})) +def check_dimension_names_length(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """One dimension name per array dimension.""" + shape = as_sequence(document["shape"]) + names = as_sequence(document["dimension_names"]) + if shape is None or names is None or len(names) == len(shape): + return () + return ( + ValidationProblem( + ("dimension_names",), + f"dimension_names has {len(names)} entries but shape has {len(shape)} dimensions", + "invalid_value", + ), + ) + + +# Generic dispatchers: every rule an entity registers for itself runs here, +# so a new codec, chunk grid, data type, or chunk key encoding needs no edit +# to this module. There must be one per extension point that has shapes: +# without it, `entity_rule` accepts a registration whose rule can never run, +# which is the silent-pass failure the registry exists to prevent. +# `test_registry.py` asserts that coverage. +_dispatch_chunk_grid = document_rule(ZARR_V3_ARRAY, frozenset({"chunk_grid"}))( + dispatch_field(CHUNK_GRID) +) +_dispatch_data_type = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( + dispatch_field(DATA_TYPE) +) +_dispatch_chunk_key_encoding = document_rule(ZARR_V3_ARRAY, frozenset({"chunk_key_encoding"}))( + dispatch_field(CHUNK_KEY_ENCODING) +) +_dispatch_codecs = document_rule(ZARR_V3_ARRAY, frozenset({"codecs"}))( + dispatch_field_sequence(CODECS) +) + + +def _rules() -> tuple[Rule, ...]: + # Importing the entity package registers every entity's rules; done here + # rather than at module import to keep the dependency one-directional. + import zarr_metadata.rules._entities as entity_rules_package + + # Imported for its registrations; referenced so the import cannot be + # pruned as unused by a checker or a well-meaning cleanup. + assert entity_rules_package is not None + return document_rules(ZARR_V3_ARRAY) + + +ZARR_V3_ARRAY_RULES: Final[tuple[Rule, ...]] = _rules() +"""The composition rule set for v3 array metadata documents. + +Assembled from the registry rather than written out, so a rule cannot be +defined without joining the set it belongs to. +""" + + +__all__ = [ + "ZARR_V3_ARRAY", + "ZARR_V3_ARRAY_RULES", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py new file mode 100644 index 0000000000..5023ef11bd --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py @@ -0,0 +1,85 @@ +"""Composition rules for v3 group metadata documents. + +A group document's own fields carry no cross-field constraints, but the +inline consolidated-metadata convention embeds whole child documents — +and a composition-invalid child makes the consolidated view lie about +the store. The group rule set therefore recurses: every array entry is +judged by the v3 array rules, and every group entry (which may itself +carry consolidated metadata) by this rule set. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from zarr_metadata.model._validation import GROUP_METADATA_STANDARD_KEYS_V3 +from zarr_metadata.rules._engine import Rule, as_string_mapping, prefixed, run_rules +from zarr_metadata.rules._registry import document_rule, document_rules, register_document_type +from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY_RULES +from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.model._validation import ValidationProblem + + +ZARR_V3_GROUP = "zarr_v3_group" +"""Document-type key under which this module's rules are registered.""" + +# `consolidated_metadata` is not a declared member of the group TypedDict: +# the spec grandfathers it as a convention that 'lacks the name member +# required of extension objects'. It is declared here so the rule that +# reads it passes the typo check without exempting unknown keys. +register_document_type( + ZARR_V3_GROUP, + GROUP_METADATA_STANDARD_KEYS_V3, + extension_keys=frozenset({ZARR_V3_CONSOLIDATED_METADATA_KEY}), +) + + +@document_rule(ZARR_V3_GROUP, frozenset({"consolidated_metadata"})) +def check_consolidated_entries(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Consolidated child documents must satisfy their own composition rules. + + Structural validity of the consolidated envelope and its entries is + the model layer's job; entries that are not interpretable as node + documents decline in its favor. + """ + return consolidated_entries_problems( + document["consolidated_metadata"], ("consolidated_metadata",) + ) + + +def consolidated_entries_problems( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[ValidationProblem, ...]: + """Composition problems in an inline consolidated envelope's children.""" + consolidated = as_string_mapping(value) + if consolidated is None: + return () + metadata = as_string_mapping(consolidated.get("metadata")) + if metadata is None: + return () + problems: list[ValidationProblem] = [] + for path, entry in metadata.items(): + node = as_string_mapping(entry) + if node is None: + continue + entry_loc = (*loc, "metadata", path) + node_type = node.get("node_type") + if node_type == "array": + problems.extend(prefixed(entry_loc, run_rules(ZARR_V3_ARRAY_RULES, node))) + elif node_type == "group": + problems.extend(prefixed(entry_loc, run_rules(ZARR_V3_GROUP_RULES, node))) + return tuple(problems) + + +ZARR_V3_GROUP_RULES: Final[tuple[Rule, ...]] = document_rules(ZARR_V3_GROUP) +"""The composition rule set for v3 group metadata documents.""" + + +__all__ = [ + "ZARR_V3_GROUP", + "ZARR_V3_GROUP_RULES", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py new file mode 100644 index 0000000000..03a8472a2b --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py @@ -0,0 +1,54 @@ +"""The Zarr v3 extension points, and how names are keyed under them. + +Names are unique only within an extension point (`bytes` is both a core +codec and a registered data type), so every table in this package is +keyed by `(field, canonical name)`. + +`canonical_name` is identity except for raw-byte data types: every `r` +spelling, valid or not, maps to `RAW_BYTES_FAMILY`, so a malformed member +of that family is reported as a misspelling rather than passing as an +unknown extension. Canonical names are lookup keys and are never emitted. +""" + +from __future__ import annotations + +from typing import Final, Literal + +from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN + +ExtensionPointField = Literal[ + "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" +] +"""The v3 array metadata fields whose values name an extension.""" + +DATA_TYPE: Final[ExtensionPointField] = "data_type" +CHUNK_GRID: Final[ExtensionPointField] = "chunk_grid" +CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" +CODECS: Final[ExtensionPointField] = "codecs" +STORAGE_TRANSFORMERS: Final[ExtensionPointField] = "storage_transformers" + +RAW_BYTES_FAMILY: Final = "r" +"""Canonical key for the parameterized raw-bytes data type family. + +Spelled as the spec writes the family; the angle brackets keep it +unforgeable by a real name. +""" + + +def canonical_name(field: ExtensionPointField, name: str) -> str: + """`name` reduced to the key this package tables it under.""" + if field == DATA_TYPE and RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None: + return RAW_BYTES_FAMILY + return name + + +__all__ = [ + "CHUNK_GRID", + "CHUNK_KEY_ENCODING", + "CODECS", + "DATA_TYPE", + "RAW_BYTES_FAMILY", + "STORAGE_TRANSFORMERS", + "ExtensionPointField", + "canonical_name", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py new file mode 100644 index 0000000000..a31cc04d52 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py @@ -0,0 +1,666 @@ +""" +Type-level shape validation for known metadata-field entities. + +One validator per extension-point entity this package defines, exact with +respect to the entity's declared TypedDicts: a value yields no problems +exactly when it is an instance of the canonical metadata type, so a +verdict here means the same thing the type does. + +Two package-wide conventions qualify "exact": + +- `int`-annotated fields mean JSON integers, so JSON booleans are + rejected even though `bool` is an `int` subtype in Python's type + system (matching the fill-value rules' treatment of integers). +- Judgments are at the canonical data level: JSON arrays are tuples, as + the TypedDicts declare. Normalize a freshly-`json.loads`-ed document + (e.g. with a model-layer parser) before asking for shape verdicts. + +Value judgments beyond the types — permutation contents, shard geometry, +cross-field consistency — belong to the composition rule layer, not here. + +Unknown names are not judged (extension openness): the `validate_known_*` +functions answer `None` for entities this package has no types for, no +problems for a valid known entity, and problems otherwise. + +Key sets are derived from the TypedDicts' `__annotations__` / +`__required_keys__` rather than restated by hand, so those entries cannot +drift from the canonical types; only the per-field value checks are +written out. The exception is `_BARE_DATA_TYPE_NAMES`: the core scalar +data types have no TypedDict to derive from — their whole metadata is a +name — so that list is hand-written, and `tests/test_registry_drift.py` +ties it to the modules that define those names. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, cast + +from zarr_metadata.model._validation import ( + ValidationProblem, + is_json, + is_metadata_field_v3, +) +from zarr_metadata.v3._extension_points import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, + RAW_BYTES_FAMILY, + ExtensionPointField, + canonical_name, +) +from zarr_metadata.v3.chunk_grid.rectilinear import ( + RECTILINEAR_CHUNK_GRID_NAME, + RectilinearChunkGridConfiguration, + RectilinearChunkGridObject, +) +from zarr_metadata.v3.chunk_grid.regular import ( + REGULAR_CHUNK_GRID_NAME, + RegularChunkGridConfiguration, + RegularChunkGridObject, +) +from zarr_metadata.v3.chunk_key_encoding.default import ( + DEFAULT_CHUNK_KEY_ENCODING_NAME, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, + DefaultChunkKeyEncodingConfiguration, + DefaultChunkKeyEncodingObject, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2_CHUNK_KEY_ENCODING_NAME, + V2_CHUNK_KEY_ENCODING_SEPARATOR, + V2ChunkKeyEncodingConfiguration, + V2ChunkKeyEncodingObject, +) +from zarr_metadata.v3.codec.blosc import ( + BLOSC_CNAME, + BLOSC_CODEC_NAME, + BLOSC_SHUFFLE, + BloscCodecConfiguration, + BloscCodecObject, +) +from zarr_metadata.v3.codec.bytes import ( + BYTES_CODEC_NAME, + ENDIANNESS, + BytesCodecConfiguration, + BytesCodecObject, +) +from zarr_metadata.v3.codec.cast_value import ( + CAST_OUT_OF_RANGE_MODE, + CAST_ROUNDING_MODE, + CAST_VALUE_CODEC_NAME, + CastValueCodecConfiguration, + CastValueCodecObject, + ScalarMap, +) +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME, Crc32cCodecObject, Empty +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME, GzipCodecConfiguration, GzipCodecObject +from zarr_metadata.v3.codec.scale_offset import ( + SCALE_OFFSET_CODEC_NAME, + ScaleOffsetCodecConfiguration, + ScaleOffsetCodecObject, +) +from zarr_metadata.v3.codec.sharding_indexed import ( + SHARDING_INDEX_LOCATION, + SHARDING_INDEXED_CODEC_NAME, + ShardingIndexedCodecConfiguration, + ShardingIndexedCodecObject, +) +from zarr_metadata.v3.codec.transpose import ( + TRANSPOSE_CODEC_NAME, + TransposeCodecConfiguration, + TransposeCodecObject, +) +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecConfiguration, ZstdCodecObject +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.numpy_datetime64 import ( + NUMPY_DATETIME64_DATA_TYPE_NAME, + NumpyDatetime64, + NumpyDatetime64Configuration, +) +from zarr_metadata.v3.data_type.numpy_timedelta64 import ( + NUMPY_TIME_UNIT, + NUMPY_TIMEDELTA64_DATA_TYPE_NAME, + NumpyTimedelta64, + NumpyTimedelta64Configuration, +) +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.struct import ( + STRUCT_DATA_TYPE_NAME, + Struct, + StructConfiguration, + StructField, +) +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME + +if TYPE_CHECKING: + from collections.abc import Callable + + from zarr_metadata.model._validation import ProblemKind + + _FieldChecker = Callable[[object, tuple[str | int, ...]], tuple[ValidationProblem, ...]] + + +def entity_name(value: object) -> str | None: + """The `name` of a metadata-field entry in any spelling, or None. + + A bare string is its own name; an object's name is its `name` member. + Anything else (or a mapping without a string `name`) has no name and + is not interpretable as a known entity. + """ + if isinstance(value, str): + return value + if not isinstance(value, Mapping): + return None + name = cast("Mapping[object, object]", value).get("name") + return name if isinstance(name, str) else None + + +def _problems( + loc: tuple[str | int, ...], message: str, kind: ProblemKind = "invalid_type" +) -> tuple[ValidationProblem, ...]: + return (ValidationProblem(loc, message, kind),) + + +def _check_json_int(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if isinstance(value, bool) or not isinstance(value, int): + return _problems(loc, f"expected an integer, got {value!r}") + return () + + +def _check_json_bool(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if not isinstance(value, bool): + return _problems(loc, f"expected a boolean, got {value!r}") + return () + + +def _literal(allowed: tuple[str, ...]) -> _FieldChecker: + def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if value not in allowed: + return _problems(loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value") + return () + + return check + + +def _check_int_tuple(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple): + return _problems(loc, f"expected an array (tuple) of integers, got {value!r}") + items = cast("tuple[object, ...]", value) + return tuple( + problem + for index, item in enumerate(items) + for problem in _check_json_int(item, (*loc, index)) + ) + + +def _check_json_value(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if not is_json(value): + return _problems(loc, "expected a JSON value") + return () + + +def _check_metadata_field( + value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + if not is_metadata_field_v3(value): + return _problems(loc, "expected a metadata field (bare name or name/configuration object)") + return () + + +def _check_field_tuple(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple): + return _problems(loc, f"expected an array (tuple) of metadata fields, got {value!r}") + items = cast("tuple[object, ...]", value) + return tuple( + problem + for index, item in enumerate(items) + for problem in _check_metadata_field(item, (*loc, index)) + ) + + +_STRUCT_FIELD_KEYS: Final = frozenset(StructField.__annotations__) + + +def _check_data_type_field( + value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + """A nested `data_type` position: structural shape plus its known shape. + + `cast_value`'s target type and a struct field's type are data types + like any other, so they get the judgment a top-level `data_type` gets. + Without this the same value is accepted in one position and rejected + in another — a bare `"numpy.datetime64"` is invalid at the top level + (its configuration is required) and was silently fine inside a struct. + Recurses naturally: a struct of structs is judged all the way down. + """ + problems = _check_metadata_field(value, loc) + if len(problems) != 0: + return problems + found = validate_known_entity_metadata(DATA_TYPE, value) + return () if found is None else _prefixed_at(loc, found) + + +def _prefixed_at( + loc: tuple[str | int, ...], problems: tuple[ValidationProblem, ...] +) -> tuple[ValidationProblem, ...]: + return tuple( + ValidationProblem((*loc, *problem.loc), problem.message, problem.kind) + for problem in problems + ) + + +def _check_struct_fields( + value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple): + return _problems(loc, f"expected an array (tuple) of struct fields, got {value!r}") + problems: list[ValidationProblem] = [] + for index, item in enumerate(cast("tuple[object, ...]", value)): + item_loc = (*loc, index) + if not isinstance(item, Mapping): + problems.extend(_problems(item_loc, f"expected an object, got {item!r}")) + continue + field = cast("Mapping[object, object]", item) + for key in field: + if not isinstance(key, str) or key not in _STRUCT_FIELD_KEYS: + problems.extend(_problems(item_loc, f"unexpected key {key!r}", "unknown_key")) + for key in sorted(_STRUCT_FIELD_KEYS - field.keys()): + problems.extend(_problems((*item_loc, key), "missing required key", "missing_key")) + if "name" in field and not isinstance(field["name"], str): + problems.extend(_problems((*item_loc, "name"), "expected a string")) + if "data_type" in field: + problems.extend(_check_data_type_field(field["data_type"], (*item_loc, "data_type"))) + return tuple(problems) + + +_SCALAR_MAP_KEYS: Final = frozenset(ScalarMap.__annotations__) + + +def _check_scalar_map_entries( + value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple): + return _problems(loc, f"expected an array (tuple) of [old, new] pairs, got {value!r}") + problems: list[ValidationProblem] = [] + for index, item in enumerate(cast("tuple[object, ...]", value)): + if not isinstance(item, tuple) or len(cast("tuple[object, ...]", item)) != 2: + problems.extend(_problems((*loc, index), f"expected an [old, new] pair, got {item!r}")) + continue + for position, scalar in enumerate(cast("tuple[object, ...]", item)): + problems.extend(_check_json_value(scalar, (*loc, index, position))) + return tuple(problems) + + +def _check_scalar_map(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + if not isinstance(value, Mapping): + return _problems(loc, f"expected an object, got {value!r}") + mapping = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + for key in mapping: + if not isinstance(key, str) or key not in _SCALAR_MAP_KEYS: + problems.extend(_problems((*loc,), f"unexpected key {key!r}", "unknown_key")) + for key in _SCALAR_MAP_KEYS: + if key in mapping: + problems.extend(_check_scalar_map_entries(mapping[key], (*loc, key))) + return tuple(problems) + + +def _check_rectilinear_dim_spec( + value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + if not isinstance(value, bool) and isinstance(value, int): + return () + if not isinstance(value, tuple): + return _problems( + loc, + f"expected an integer or an array of integers / [value, count] pairs, got {value!r}", + ) + problems: list[ValidationProblem] = [] + for index, item in enumerate(cast("tuple[object, ...]", value)): + if not isinstance(item, bool) and isinstance(item, int): + continue + if isinstance(item, tuple) and len(cast("tuple[object, ...]", item)) == 2: + problems.extend( + problem + for position, part in enumerate(cast("tuple[object, ...]", item)) + for problem in _check_json_int(part, (*loc, index, position)) + ) + continue + problems.extend( + _problems((*loc, index), f"expected an integer or a [value, count] pair, got {item!r}") + ) + return tuple(problems) + + +def _check_rectilinear_dim_specs( + value: object, loc: tuple[str | int, ...] +) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple): + return _problems(loc, f"expected an array (tuple) of dimension specs, got {value!r}") + return tuple( + problem + for index, item in enumerate(cast("tuple[object, ...]", value)) + for problem in _check_rectilinear_dim_spec(item, (*loc, index)) + ) + + +@dataclass(frozen=True, slots=True) +class _EntityShape: + """Shape facts for one known entity name, derived from its TypedDicts.""" + + object_keys: frozenset[str] + configuration_required: bool + config_keys: frozenset[str] + config_required: frozenset[str] + config_checkers: Mapping[str, _FieldChecker] + + +def _shape( + object_type: type, + configuration_type: type, + checkers: Mapping[str, _FieldChecker], +) -> _EntityShape: + config_keys = frozenset(configuration_type.__annotations__) + if frozenset(checkers) != config_keys: + raise AssertionError( # pragma: no cover - registry construction guard + f"checkers {sorted(checkers)} do not cover configuration keys {sorted(config_keys)}" + ) + return _EntityShape( + object_keys=frozenset(object_type.__annotations__), + configuration_required="configuration" + in cast("frozenset[str]", object_type.__required_keys__), # type: ignore[attr-defined] + config_keys=config_keys, + config_required=cast( + "frozenset[str]", + configuration_type.__required_keys__, # type: ignore[attr-defined] + ), + config_checkers=dict(checkers), + ) + + +def _bare_shape() -> _EntityShape: + """Shape of an entity that takes no configuration. + + Both spellings are valid: the spec makes `{"name": ...}` the base form + and permits the bare short-hand when no configuration is required. The + object form is therefore accepted with an absent or empty + `configuration`, and any member inside one is an unknown key. + + Keyword arguments deliberately: this is a six-field record whose flags + are easy to transpose positionally. + """ + return _EntityShape( + object_keys=frozenset({"name", "configuration", "must_understand"}), + configuration_required=False, + config_keys=frozenset(), + config_required=frozenset(), + config_checkers={}, + ) + + +_CODEC_SHAPES: Final[Mapping[str, _EntityShape]] = { + BLOSC_CODEC_NAME: _shape( + BloscCodecObject, + BloscCodecConfiguration, + { + "cname": _literal(BLOSC_CNAME), + "clevel": _check_json_int, + "shuffle": _literal(BLOSC_SHUFFLE), + "blocksize": _check_json_int, + "typesize": _check_json_int, + }, + ), + BYTES_CODEC_NAME: _shape( + BytesCodecObject, BytesCodecConfiguration, {"endian": _literal(ENDIANNESS)} + ), + CAST_VALUE_CODEC_NAME: _shape( + CastValueCodecObject, + CastValueCodecConfiguration, + { + "data_type": _check_data_type_field, + "rounding": _literal(CAST_ROUNDING_MODE), + "out_of_range": _literal(CAST_OUT_OF_RANGE_MODE), + "scalar_map": _check_scalar_map, + }, + ), + CRC32C_CODEC_NAME: _shape(Crc32cCodecObject, Empty, {}), + GZIP_CODEC_NAME: _shape(GzipCodecObject, GzipCodecConfiguration, {"level": _check_json_int}), + SCALE_OFFSET_CODEC_NAME: _shape( + ScaleOffsetCodecObject, + ScaleOffsetCodecConfiguration, + {"offset": _check_json_value, "scale": _check_json_value}, + ), + SHARDING_INDEXED_CODEC_NAME: _shape( + ShardingIndexedCodecObject, + ShardingIndexedCodecConfiguration, + { + "chunk_shape": _check_int_tuple, + "codecs": _check_field_tuple, + "index_codecs": _check_field_tuple, + "index_location": _literal(SHARDING_INDEX_LOCATION), + }, + ), + TRANSPOSE_CODEC_NAME: _shape( + TransposeCodecObject, TransposeCodecConfiguration, {"order": _check_int_tuple} + ), + ZSTD_CODEC_NAME: _shape( + ZstdCodecObject, + ZstdCodecConfiguration, + {"level": _check_json_int, "checksum": _check_json_bool}, + ), +} + +_CHUNK_GRID_SHAPES: Final[Mapping[str, _EntityShape]] = { + REGULAR_CHUNK_GRID_NAME: _shape( + RegularChunkGridObject, RegularChunkGridConfiguration, {"chunk_shape": _check_int_tuple} + ), + RECTILINEAR_CHUNK_GRID_NAME: _shape( + RectilinearChunkGridObject, + RectilinearChunkGridConfiguration, + { + # "inline" is the sole member of the kind Literal; the type + # exports no constant tuple for it. + "kind": _literal(("inline",)), + "chunk_shapes": _check_rectilinear_dim_specs, + }, + ), +} + +_CHUNK_KEY_ENCODING_SHAPES: Final[Mapping[str, _EntityShape]] = { + DEFAULT_CHUNK_KEY_ENCODING_NAME: _shape( + DefaultChunkKeyEncodingObject, + DefaultChunkKeyEncodingConfiguration, + {"separator": _literal(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)}, + ), + V2_CHUNK_KEY_ENCODING_NAME: _shape( + V2ChunkKeyEncodingObject, + V2ChunkKeyEncodingConfiguration, + {"separator": _literal(V2_CHUNK_KEY_ENCODING_SEPARATOR)}, + ), +} + +_BARE_DATA_TYPE_NAMES: Final = ( + BOOL_DATA_TYPE_NAME, + INT8_DATA_TYPE_NAME, + INT16_DATA_TYPE_NAME, + INT32_DATA_TYPE_NAME, + INT64_DATA_TYPE_NAME, + UINT8_DATA_TYPE_NAME, + UINT16_DATA_TYPE_NAME, + UINT32_DATA_TYPE_NAME, + UINT64_DATA_TYPE_NAME, + FLOAT16_DATA_TYPE_NAME, + FLOAT32_DATA_TYPE_NAME, + FLOAT64_DATA_TYPE_NAME, + COMPLEX64_DATA_TYPE_NAME, + COMPLEX128_DATA_TYPE_NAME, + RAW_BYTES_FAMILY, + BYTES_DATA_TYPE_NAME, + STRING_DATA_TYPE_NAME, +) + +_DATA_TYPE_SHAPES: Final[Mapping[str, _EntityShape]] = { + **{name: _bare_shape() for name in _BARE_DATA_TYPE_NAMES}, + NUMPY_DATETIME64_DATA_TYPE_NAME: _shape( + NumpyDatetime64, + NumpyDatetime64Configuration, + {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _check_json_int}, + ), + NUMPY_TIMEDELTA64_DATA_TYPE_NAME: _shape( + NumpyTimedelta64, + NumpyTimedelta64Configuration, + {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _check_json_int}, + ), + STRUCT_DATA_TYPE_NAME: _shape(Struct, StructConfiguration, {"fields": _check_struct_fields}), +} + + +def _validate_known_entity( + value: object, name: str, shape: _EntityShape, entity: str +) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not an instance of `name`'s canonical type. + + Locations are relative to the entry itself (`("configuration", key)` + etc.); callers prefix the entry's position in its document. + """ + if isinstance(value, str): + if not shape.configuration_required: + return () + return _problems( + (), + f"{entity} {name!r} requires a configuration and has no bare short-hand " + f"form; use {{'name': {name!r}, 'configuration': {{...}}}}", + "invalid_value", + ) + if not isinstance(value, Mapping): + return _problems((), f"expected a bare name or an object, got {value!r}") + mapping = cast("Mapping[object, object]", value) + problems: list[ValidationProblem] = [] + for key in mapping: + if not isinstance(key, str) or key not in shape.object_keys: + problems.extend(_problems((), f"unexpected key {key!r}", "unknown_key")) + if "must_understand" in mapping: + problems.extend(_check_json_bool(mapping["must_understand"], ("must_understand",))) + if "configuration" not in mapping: + if shape.configuration_required: + problems.extend( + _problems( + ("configuration",), + f"{entity} {name!r} requires a 'configuration' object", + "missing_key", + ) + ) + return tuple(problems) + configuration = mapping["configuration"] + if not isinstance(configuration, Mapping): + problems.extend(_problems(("configuration",), f"expected an object, got {configuration!r}")) + return tuple(problems) + config = cast("Mapping[object, object]", configuration) + for key in config: + if not isinstance(key, str) or key not in shape.config_keys: + problems.extend(_problems(("configuration",), f"unexpected key {key!r}", "unknown_key")) + for key in sorted(shape.config_required - {k for k in config if isinstance(k, str)}): + problems.extend( + _problems( + ("configuration", key), + f"configuration for {entity} {name!r} is missing required key {key!r}", + "missing_key", + ) + ) + for key, checker in shape.config_checkers.items(): + if key in config: + problems.extend(checker(config[key], ("configuration", key))) + return tuple(problems) + + +def validate_known_codec_metadata(value: object) -> tuple[ValidationProblem, ...] | None: + """Shape problems for a known-name codec entry, or None if not judged. + + None means the entry has no interpretable name or its name is not a + codec this package defines (extension openness: unknown entities are + not ours to judge). An empty list means `value` is an instance of the + named codec's canonical metadata type. + """ + name = entity_name(value) + if name is None: + return None + shape = _CODEC_SHAPES.get(name) + if shape is None: + return None + return _validate_known_entity(value, name, shape, "codec") + + +def validate_known_chunk_grid_metadata(value: object) -> tuple[ValidationProblem, ...] | None: + """Shape problems for a known-name chunk grid entry, or None if not judged.""" + name = entity_name(value) + if name is None: + return None + shape = _CHUNK_GRID_SHAPES.get(name) + if shape is None: + return None + return _validate_known_entity(value, name, shape, "chunk grid") + + +_ENTITY_SHAPES: Final[Mapping[ExtensionPointField, Mapping[str, _EntityShape]]] = { + DATA_TYPE: _DATA_TYPE_SHAPES, + CODECS: _CODEC_SHAPES, + CHUNK_GRID: _CHUNK_GRID_SHAPES, + CHUNK_KEY_ENCODING: _CHUNK_KEY_ENCODING_SHAPES, +} + + +def validate_known_entity_metadata( + field: ExtensionPointField, value: object +) -> tuple[ValidationProblem, ...] | None: + """Shape problems for an entity known at `field`, or None if not judged.""" + name = entity_name(value) + if name is None: + return None + shape = _ENTITY_SHAPES.get(field, {}).get(canonical_name(field, name)) + if shape is None: + return None + return _validate_known_entity(value, name, shape, field.replace("_", " ").rstrip("s")) + + +def modelled_entities() -> frozenset[tuple[ExtensionPointField, str]]: + """Every `(extension point, name)` with a shape validator.""" + return frozenset((field, name) for field, shapes in _ENTITY_SHAPES.items() for name in shapes) + + +def blocking_problems( + problems: Sequence[ValidationProblem], +) -> tuple[ValidationProblem, ...]: + """The problems that prevent interpreting an entity's fields. + + `unknown_key` problems do not: a member this package does not model + says nothing about the members it does. Rules use this so a single + unrecognized key cannot silently suppress every other judgment about + the same entity — the extra key is still reported, and the geometry + checks still run. + """ + return tuple(problem for problem in problems if problem.kind != "unknown_key") + + +__all__ = [ + "blocking_problems", + "entity_name", + "modelled_entities", + "validate_known_chunk_grid_metadata", + "validate_known_codec_metadata", + "validate_known_entity_metadata", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index c8a9a150fc..f22a2280f9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -14,6 +14,9 @@ `codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. +The `kind` submodule sorts the known codec names into the spec's three +pipeline kinds (`array -> array`, `array -> bytes`, `bytes -> bytes`). + See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ @@ -22,19 +25,31 @@ from zarr_metadata.v3.codec.cast_value import CastValueCodecMetadata from zarr_metadata.v3.codec.crc32c import Crc32cCodecMetadata from zarr_metadata.v3.codec.gzip import GzipCodecMetadata +from zarr_metadata.v3.codec.kind import ( + ARRAY_ARRAY_CODEC_NAMES, + ARRAY_BYTES_CODEC_NAMES, + BYTES_BYTES_CODEC_NAMES, + CodecKind, + codec_kind_of_name, +) from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodecMetadata from zarr_metadata.v3.codec.sharding_indexed import ShardingIndexedCodecMetadata from zarr_metadata.v3.codec.transpose import TransposeCodecMetadata from zarr_metadata.v3.codec.zstd import ZstdCodecMetadata __all__ = [ + "ARRAY_ARRAY_CODEC_NAMES", + "ARRAY_BYTES_CODEC_NAMES", + "BYTES_BYTES_CODEC_NAMES", "BloscCodecMetadata", "BytesCodecMetadata", "CastValueCodecMetadata", + "CodecKind", "Crc32cCodecMetadata", "GzipCodecMetadata", "ScaleOffsetCodecMetadata", "ShardingIndexedCodecMetadata", "TransposeCodecMetadata", "ZstdCodecMetadata", + "codec_kind_of_name", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py new file mode 100644 index 0000000000..a5853c0f97 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py @@ -0,0 +1,66 @@ +"""Classify Zarr v3 codecs by pipeline kind. + +The v3 spec sorts codecs into three kinds — `array -> array`, +`array -> bytes`, `bytes -> bytes` — and a pipeline is +`array->array* array->bytes bytes->bytes*`. `codec_kind_of_name` +classifies a known name; unknown names have no kind. + +See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html +""" + +from typing import Final, Literal + +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME +from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME +from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC_NAME +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME + +ARRAY_ARRAY_CODEC_NAMES: Final = ( + TRANSPOSE_CODEC_NAME, + CAST_VALUE_CODEC_NAME, + SCALE_OFFSET_CODEC_NAME, +) +"""Tuple of the `name` field values of the known `array -> array` codecs.""" + +ARRAY_BYTES_CODEC_NAMES: Final = (BYTES_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME) +"""Tuple of the `name` field values of the known `array -> bytes` codecs.""" + +BYTES_BYTES_CODEC_NAMES: Final = ( + BLOSC_CODEC_NAME, + CRC32C_CODEC_NAME, + GZIP_CODEC_NAME, + ZSTD_CODEC_NAME, +) +"""Tuple of the `name` field values of the known `bytes -> bytes` codecs.""" + +CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] +"""The three pipeline positions the v3 spec sorts codecs into.""" + + +def codec_kind_of_name(name: str) -> CodecKind | None: + """The pipeline kind of the codec named `name`, or None if unknown. + + Classifies by name alone, with no judgment of the entry's spelling or + configuration; the rules layer judges those separately. + """ + if name in ARRAY_ARRAY_CODEC_NAMES: + return "array_array" + if name in ARRAY_BYTES_CODEC_NAMES: + return "array_bytes" + if name in BYTES_BYTES_CODEC_NAMES: + return "bytes_bytes" + return None + + +__all__ = [ + "ARRAY_ARRAY_CODEC_NAMES", + "ARRAY_BYTES_CODEC_NAMES", + "BYTES_BYTES_CODEC_NAMES", + "CodecKind", + "codec_kind_of_name", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 66e69c9b53..369dc282c1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -18,7 +18,14 @@ https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 """ -_RAW_BYTES_RE: Final = re.compile(r"^r(\d+)$") +RAW_BYTES_NAME_PATTERN: Final = re.compile(r"^r(\d+)$") +"""The *shape* of a raw-bytes data type name, not its validity. + +Matches every `r` spelling including malformed ones (`r0`, `r12`), so +that a misspelled member of this family is recognized as belonging to it +and reported as a misspelling, rather than passing as an unknown +third-party extension. `raw_bytes_dtype_name` applies the validity rule +on top. Sole owner of this grammar: other modules match through it.""" def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: @@ -27,7 +34,7 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: Raises ValueError if `value` is not `r` followed by a positive multiple of 8. """ - match = _RAW_BYTES_RE.fullmatch(value) + match = RAW_BYTES_NAME_PATTERN.fullmatch(value) if match is None: raise ValueError(f"Expected 'r' followed by a positive integer, got {value!r}") bits = int(match.group(1)) @@ -44,6 +51,7 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: __all__ = [ + "RAW_BYTES_NAME_PATTERN", "RawBytesDataTypeName", "RawBytesFillValue", "raw_bytes_dtype_name", diff --git a/packages/zarr-metadata/tests/builder/__init__.py b/packages/zarr-metadata/tests/builder/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/builder/test_create.py b/packages/zarr-metadata/tests/builder/test_create.py new file mode 100644 index 0000000000..f40cd56206 --- /dev/null +++ b/packages/zarr-metadata/tests/builder/test_create.py @@ -0,0 +1,254 @@ +"""Tests for the `create_*` document factories.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +import typing_extensions + +import zarr_metadata +import zarr_metadata.builder +from zarr_metadata.builder._create import ( + create_zarr_v2_array_metadata_json, + create_zarr_v2_consolidated_metadata_json, + create_zarr_v2_group_metadata_json, + create_zarr_v2_zarray_json, + create_zarr_v2_zgroup_json, + create_zarr_v3_array_metadata_json, + create_zarr_v3_consolidated_metadata_json, + create_zarr_v3_group_metadata_json, +) +from zarr_metadata.model import MetadataValidationError + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + # Factories differ in the document type they return; these tests only + # ever compare the result as a mapping. + Factory = Callable[..., Mapping[str, object]] + +V3_ARRAY: dict[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), +} + +V2_ZARRAY: dict[str, object] = { + "zarr_format": 2, + "shape": (4,), + "chunks": (2,), + "dtype": " None: + assert factory(**kwargs) == expected + + +def test_output_shares_no_state_with_arguments() -> None: + grid: dict[str, object] = {"name": "regular", "configuration": {"chunk_shape": (2, 2)}} + document = create_zarr_v3_array_metadata_json(**{**V3_ARRAY, "chunk_grid": grid}) + grid["configuration"]["chunk_shape"] = (9, 9) # caller mutates after the fact + assert document["chunk_grid"]["configuration"]["chunk_shape"] == (2, 2) + + +# -- the package rule, enforced ---------------------------------------------- + +# Public TypedDicts ending in JSON that are field/helper shapes rather than +# documents. Closed by hand, like the naming-grammar vocabulary. +_HELPER_SHAPES = frozenset({"ZarrV3NamedConfigJSON"}) + +# Every public document TypedDict, mapped to its factory. Kept here rather +# than in the package: the mapping exists only so this test can hold the +# two sets equal. +_FACTORIES: dict[str, Factory] = { + "ZarrV2ArrayMetadataJSON": create_zarr_v2_array_metadata_json, + "ZarrV2ConsolidatedMetadataJSON": create_zarr_v2_consolidated_metadata_json, + "ZarrV2GroupMetadataJSON": create_zarr_v2_group_metadata_json, + "ZarrV2ZArrayJSON": create_zarr_v2_zarray_json, + "ZarrV2ZGroupJSON": create_zarr_v2_zgroup_json, + "ZarrV3ArrayMetadataJSON": create_zarr_v3_array_metadata_json, + "ZarrV3ConsolidatedMetadataJSON": create_zarr_v3_consolidated_metadata_json, + "ZarrV3GroupMetadataJSON": create_zarr_v3_group_metadata_json, +} + + +def test_every_document_typeddict_has_a_factory() -> None: + documents = { + name + for name in zarr_metadata.__all__ + if typing_extensions.is_typeddict(getattr(zarr_metadata, name)) + and name.endswith("JSON") + and name not in _HELPER_SHAPES + } + assert documents == set(_FACTORIES) + for factory in _FACTORIES.values(): + assert factory.__name__ in zarr_metadata.builder.__all__ + + +# -- error cases, one test per failure mode ---------------------------------- + + +def test_error_v3_array_semantic_rules_run() -> None: + with pytest.raises(MetadataValidationError, match=r"\[0, 255\]"): + create_zarr_v3_array_metadata_json(**{**V3_ARRAY, "fill_value": 300}) + + +def test_error_v3_array_structural_garbage_from_untyped_caller() -> None: + with pytest.raises(MetadataValidationError) as info: + create_zarr_v3_array_metadata_json(zarr_format="3") # type: ignore[arg-type] + assert {p.loc[0] for p in info.value.problems if p.kind == "missing_key"} >= { + "node_type", + "shape", + } + + +def test_error_v3_array_extension_shadows_standard_key() -> None: + with pytest.raises(MetadataValidationError, match="standard metadata key"): + create_zarr_v3_array_metadata_json(**V3_ARRAY, extensions={"shape": (9,)}) + + +def test_error_v3_group_extension_shadows_standard_key() -> None: + with pytest.raises(MetadataValidationError, match="standard metadata key"): + create_zarr_v3_group_metadata_json( + zarr_format=3, node_type="group", extensions={"attributes": {}} + ) + + +def test_error_v3_consolidated_invalid() -> None: + with pytest.raises(MetadataValidationError): + create_zarr_v3_consolidated_metadata_json( + kind="inline", + must_understand=True, + metadata={}, # type: ignore[typeddict-item] + ) + + +def test_error_v3_consolidated_child_violates_composition_rules() -> None: + child = {**V3_ARRAY, "fill_value": 300} + with pytest.raises(MetadataValidationError) as exc_info: + create_zarr_v3_consolidated_metadata_json( + kind="inline", must_understand=False, metadata={"a": child} + ) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("metadata", "a", "fill_value"), "invalid_value") + ] + + +def test_error_v2_array_structural() -> None: + with pytest.raises(MetadataValidationError): + create_zarr_v2_array_metadata_json(**{**V2_ZARRAY, "order": "K"}) # type: ignore[typeddict-item] + + +def test_error_v3_array_malformed_raw_dtype() -> None: + # r names outside the family grammar are misspellings of a known + # family, not unknown extensions, and must not escape judgment. + with pytest.raises(MetadataValidationError, match="positive multiple of 8"): + create_zarr_v3_array_metadata_json(**{**V3_ARRAY, "data_type": "r12", "fill_value": (1,)}) + + +def test_error_v2_zarray_attributes_via_splat() -> None: + # The signature excludes `attributes` statically, but a splatted call + # bypasses that; the runtime backstop must hold the strict shape. + with pytest.raises(MetadataValidationError, match=".zattrs"): + create_zarr_v2_zarray_json(**{**V2_ZARRAY, "attributes": {"unit": "m"}}) + + +def test_error_v2_zgroup_attributes_via_splat() -> None: + splatted: dict[str, object] = {"zarr_format": 2, "attributes": {"unit": "m"}} + with pytest.raises(MetadataValidationError, match=".zattrs"): + create_zarr_v2_zgroup_json(**splatted) + + +def test_error_v2_consolidated_envelope() -> None: + with pytest.raises(MetadataValidationError, match="expected a mapping"): + create_zarr_v2_consolidated_metadata_json( + zarr_consolidated_format=1, + metadata="not a mapping", # type: ignore[typeddict-item] + ) + + +def test_error_v2_consolidated_format_is_not_one() -> None: + with pytest.raises(MetadataValidationError) as exc_info: + create_zarr_v2_consolidated_metadata_json( + zarr_consolidated_format=2, + metadata={}, + ) + assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ + (("zarr_consolidated_format",), "invalid_value") + ] + + +def test_error_v2_consolidated_array_entry_is_invalid() -> None: + with pytest.raises(MetadataValidationError) as exc_info: + create_zarr_v2_consolidated_metadata_json( + zarr_consolidated_format=1, + metadata={"foo/.zarray": {}}, # type: ignore[typeddict-item] + ) + assert any( + problem.loc[:2] == ("metadata", "foo/.zarray") and problem.kind == "missing_key" + for problem in exc_info.value.problems + ) + + +def test_error_v2_consolidated_entry_has_unknown_suffix() -> None: + with pytest.raises(MetadataValidationError, match="metadata file suffix"): + create_zarr_v2_consolidated_metadata_json( + zarr_consolidated_format=1, + metadata={"foo/data": {}}, # type: ignore[typeddict-item] + ) diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 69a3823b6f..408308de59 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1325,16 +1325,23 @@ def test_v2_filters_must_be_codec_sequence_or_none() -> None: assert [(p.loc, p.kind) for p in problems] == [(("filters",), "invalid_type")], bad -def test_v2_shape_and_chunks_must_have_equal_rank() -> None: - """Raw v2 metadata requires one chunk length per array dimension.""" +def test_v2_shape_chunks_rank_agreement_is_not_structural() -> None: + """Whether chunks matches shape's dimensionality is a composition + judgment owned by zarr_metadata.rules; the structural validator and the + model classes deliberately accept the document (it is a lossless, + structurally well-formed representation of what a store may contain).""" doc = dict(ZarrV2ArrayMetadata.create_default(shape=(2, 3)).to_json()) doc["chunks"] = (1,) - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ + assert validate_array_metadata_v2(doc) == () + parsed = ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + assert parsed.chunks == (1,) + + from zarr_metadata import rules + + assert [(p.loc, p.kind) for p in rules.validate_array_metadata_v2(doc)] == [ (("chunks",), "invalid_value") ] - with pytest.raises(MetadataValidationError, match="same number of dimensions"): - ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) def test_v2_filters_may_be_empty() -> None: @@ -1527,12 +1534,18 @@ def test_shape_rejects_negative_dimensions() -> None: ] -def test_dimension_names_length_must_match_shape() -> None: - """dimension_names must have one entry per dimension of shape.""" +def test_dimension_names_length_is_not_structural() -> None: + """Whether dimension_names matches shape's dimensionality is a + composition judgment owned by zarr_metadata.rules; the structural + validator deliberately accepts the document.""" doc = dict(ZarrV3ArrayMetadata.create_default(shape=(10,)).to_json()) | { "dimension_names": ("x", "y", "z") } - assert [(p.loc, p.kind) for p in validate_array_metadata_v3(doc)] == [ + assert validate_array_metadata_v3(doc) == () + + from zarr_metadata import rules + + assert [(p.loc, p.kind) for p in rules.validate_array_metadata_v3(doc)] == [ (("dimension_names",), "invalid_value") ] diff --git a/packages/zarr-metadata/tests/rules/__init__.py b/packages/zarr-metadata/tests/rules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/zarr-metadata/tests/rules/test_documents.py b/packages/zarr-metadata/tests/rules/test_documents.py new file mode 100644 index 0000000000..94a49b6171 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_documents.py @@ -0,0 +1,127 @@ +"""Tests for the whole-document validation trios in `zarr_metadata.rules`.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from zarr_metadata.model import MetadataValidationError +from zarr_metadata.model import ( + is_array_metadata_v3 as model_is_array_metadata_v3, +) +from zarr_metadata.rules import ( + is_array_metadata_v2, + is_array_metadata_v3, + parse_array_metadata_v2, + parse_array_metadata_v3, + validate_array_metadata_v2, + validate_array_metadata_v3, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata import ZarrV2ArrayMetadataJSON, ZarrV3ArrayMetadataJSON + from zarr_metadata.model import ValidationProblem + + # The trios are uniform in their inputs (any object) and differ only in + # the document type they hand back, which these tests never depend on. + Validator = Callable[[object], tuple[ValidationProblem, ...]] + Parser = Callable[[object], Mapping[str, object]] + Check = Callable[[object], bool] + +V3_ARRAY: ZarrV3ArrayMetadataJSON = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), +} + +V2_ARRAY: ZarrV2ArrayMetadataJSON = { + "zarr_format": 2, + "shape": (4,), + "chunks": (2,), + "dtype": " None: + parsed = parse(doc) + assert validate(parsed) == () + assert check(parsed) is True + shape = doc["shape"] + assert isinstance(shape, (list, tuple)) + assert parsed["shape"] == tuple(shape) + + +def test_error_v3_combined_report() -> None: + # One raise carrying problems from both passes: a structural problem + # (bad node_type) and a composition problem (fill_value vs data_type). + with pytest.raises(MetadataValidationError) as info: + parse_array_metadata_v3({**V3_ARRAY, "node_type": "grid", "fill_value": 300}) + kinds = {(p.loc, p.kind) for p in info.value.problems} + assert (("node_type",), "invalid_value") in kinds + assert (("fill_value",), "invalid_value") in kinds + + +def test_error_v3_dimension_names_reported_once() -> None: + # Regression: this fault used to be reported twice — once by the + # structural validator, once by the composition rule. The check now + # has one owner. + problems = validate_array_metadata_v3({**V3_ARRAY, "dimension_names": ("x",)}) + assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_value")] + + +def test_error_v2_chunks_rank() -> None: + problems = validate_array_metadata_v2({**V2_ARRAY, "chunks": (2, 2)}) + assert [(p.loc, p.kind) for p in problems] == [(("chunks",), "invalid_value")] + + +def test_error_v2_parse_raises() -> None: + with pytest.raises(MetadataValidationError, match="same number of dimensions"): + parse_array_metadata_v2({**V2_ARRAY, "chunks": (2, 2)}) + + +def test_is_functions_are_not_type_guards() -> None: + # A composition-invalid document is still an instance of the TypedDict, + # so the model layer's TypeIs narrows it while the rules layer's plain + # bool judges it. Divergence here is the design, not a bug. + doc = {**V3_ARRAY, "fill_value": 300} + assert model_is_array_metadata_v3(doc) is True + assert is_array_metadata_v3(doc) is False diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py new file mode 100644 index 0000000000..2f748a70c1 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -0,0 +1,172 @@ +"""Tests for rule registration. + +The registry exists so that a rule cannot be defined without being run. +These tests cover the three ways that could still fail: a rule declaring +dependencies no such document has, an entity whose rules were never +imported, and a document rule set assembled from something other than +the registry. +""" + +from __future__ import annotations + +import pkgutil + +import pytest + +import zarr_metadata.rules._entities as entities +from zarr_metadata.rules import ZARR_V2_ARRAY_RULES, ZARR_V3_ARRAY_RULES, ZARR_V3_GROUP_RULES +from zarr_metadata.rules._registry import ( + dispatched_fields, + document_rule, + entity_rule, + register_document_type, + registered_entities, +) +from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY +from zarr_metadata.v3._extension_points import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, + RAW_BYTES_FAMILY, +) +from zarr_metadata.v3._shape import modelled_entities + +# Entities the package models but that carry no composition rules: their +# canonical shape is the whole of what we can say about them. Listed by +# hand, keyed by extension point, so that adding a codec is a deliberate +# choice between "write rules" and "record that there are none", never a +# silent omission. +_RULE_FREE = frozenset( + { + (CODECS, "blosc"), + (CODECS, "cast_value"), + (CODECS, "crc32c"), + (CODECS, "scale_offset"), + (CODECS, "zstd"), + (CHUNK_KEY_ENCODING, "default"), + (CHUNK_KEY_ENCODING, "v2"), + (DATA_TYPE, "bool"), + (DATA_TYPE, "int8"), + (DATA_TYPE, "int16"), + (DATA_TYPE, "int32"), + (DATA_TYPE, "int64"), + (DATA_TYPE, "uint8"), + (DATA_TYPE, "uint16"), + (DATA_TYPE, "uint32"), + (DATA_TYPE, "uint64"), + (DATA_TYPE, "float16"), + (DATA_TYPE, "float32"), + (DATA_TYPE, "float64"), + (DATA_TYPE, "complex64"), + (DATA_TYPE, "complex128"), + (DATA_TYPE, RAW_BYTES_FAMILY), + (DATA_TYPE, "bytes"), + (DATA_TYPE, "string"), + } +) + + +def test_every_shape_modelled_entity_is_accounted_for() -> None: + # Every shape-modelled entity either + # carries rules or is recorded as deliberately rule-free. + assert modelled_entities() == registered_entities() | _RULE_FREE + + +def test_every_shape_modelled_field_has_a_dispatcher() -> None: + # Regression: shapes existed for four extension points but dispatchers + # for only two, so `entity_rule` accepted registrations at `data_type` + # and `chunk_key_encoding` whose rules then silently never ran — the + # exact silent-pass failure the registry exists to prevent. A rule can + # only fire at a field something dispatches. + shape_modelled = {field for field, _ in modelled_entities()} + assert shape_modelled <= dispatched_fields() + + +def test_every_field_with_rules_has_a_dispatcher() -> None: + assert {field for field, _ in registered_entities()} <= dispatched_fields() + + +def test_rule_free_entities_really_have_no_rules() -> None: + # Guards the exclusion list itself: an entity cannot be listed as + # rule-free while quietly carrying rules. + assert registered_entities() & _RULE_FREE == frozenset() + + +def test_rules_are_keyed_by_extension_point_not_name() -> None: + # `bytes` is a core codec and a registered extension data type; a rule + # for one must never fire on the other, so the key carries the field. + assert {(CODECS, "bytes"), (DATA_TYPE, "bytes")} <= modelled_entities() + assert (CODECS, "bytes") in registered_entities() + assert (DATA_TYPE, "bytes") not in registered_entities() + + +def test_every_entity_module_is_imported() -> None: + # The package auto-imports its modules; this asserts the discovery + # actually ran, so a new module cannot sit unimported and inert. + module_names = {info.name for info in pkgutil.iter_modules(entities.__path__)} + assert len(module_names) != 0 + for name in module_names: + assert f"{entities.__name__}.{name}" in __import__("sys").modules + + +@pytest.mark.parametrize("rules", [ZARR_V3_ARRAY_RULES, ZARR_V2_ARRAY_RULES, ZARR_V3_GROUP_RULES]) +def test_rule_sets_are_non_empty(rules: tuple[object, ...]) -> None: + assert len(rules) != 0 + + +def test_error_document_rule_requiring_an_unknown_key() -> None: + # A rule whose dependency is misspelled can never fire, and a rule + # that never fires is indistinguishable from one that always passes. + with pytest.raises(ValueError, match="could never fire"): + + @document_rule(ZARR_V3_ARRAY, frozenset({"shapee"})) + def _misspelled(document: object) -> tuple[()]: # pragma: no cover - never runs + return () + + +def test_error_entity_rule_requiring_an_unknown_key() -> None: + with pytest.raises(ValueError, match="could never fire"): + + @entity_rule(ZARR_V3_ARRAY, CHUNK_GRID, "regular", requires=frozenset({"shapee"})) + def _misspelled(configuration: object, document: object) -> tuple[()]: # pragma: no cover + return () + + +def test_error_entity_rule_for_an_unmodelled_entity() -> None: + # Entity rules read configuration members by name, so a rule for an + # entity with no shape validator could never fire. + with pytest.raises(ValueError, match="no shape validator"): + + @entity_rule(ZARR_V3_ARRAY, CHUNK_GRID, "hilbert") + def _unmodelled(configuration: object, document: object) -> tuple[()]: # pragma: no cover + return () + + +def test_error_entity_rule_for_name_modelled_only_at_another_extension_point() -> None: + # `regular` has a chunk-grid shape, but no codec shape. Name-only lookup + # would accept this registration and later interpret codec metadata using + # the chunk-grid schema. + with pytest.raises(ValueError, match="no shape validator"): + + @entity_rule(ZARR_V3_ARRAY, CODECS, "regular") + def _wrong_extension_point(configuration: object, document: object) -> tuple[()]: + return () + + +def test_error_rule_for_an_unregistered_document_type() -> None: + with pytest.raises(LookupError, match="unknown document type"): + + @document_rule("zarr_v9_array", frozenset()) + def _orphan(document: object) -> tuple[()]: # pragma: no cover - never runs + return () + + +def test_register_document_type_accepts_declared_extension_keys() -> None: + register_document_type("test_doc", frozenset({"a"}), extension_keys=frozenset({"b"})) + + @document_rule("test_doc", frozenset({"a", "b"})) + def _uses_both(document: object) -> tuple[()]: + return () + + assert _uses_both.requires == frozenset({"a", "b"}) diff --git a/packages/zarr-metadata/tests/rules/test_result.py b/packages/zarr-metadata/tests/rules/test_result.py new file mode 100644 index 0000000000..49e859a873 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_result.py @@ -0,0 +1,107 @@ +"""Tests for the `check_*` tagged-union entry points.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, get_args, get_type_hints + +import pytest + +from zarr_metadata.rules import ( + Invalid, + Valid, + ValidationResult, + check_array_metadata_v2, + check_array_metadata_v3, + check_group_metadata_v2, + check_group_metadata_v3, +) +from zarr_metadata.rules._documents import validate_array_metadata_v3 + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + +V3_ARRAY: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), +} +V2_ARRAY: Mapping[str, object] = { + "zarr_format": 2, + "shape": (4,), + "chunks": (2,), + "dtype": " None: + result = check(doc) + assert isinstance(result, Valid) + assert result.valid is True + assert result.document == doc + + +def test_valid_normalizes_like_parse() -> None: + # A Valid carries the canonical document, not the caller's spelling. + result = check_array_metadata_v3({**V3_ARRAY, "shape": [4, 4], "codecs": ["bytes"]}) + assert isinstance(result, Valid) + assert result.document["shape"] == (4, 4) + assert result.document["codecs"] == ("bytes",) + + +def test_error_invalid_carries_every_problem() -> None: + result = check_array_metadata_v3({**V3_ARRAY, "node_type": "grid", "fill_value": 300}) + assert isinstance(result, Invalid) + assert result.valid is False + assert len(result.problems) != 0 + # the same report validate_* would give, not a summary of it + assert result.problems == validate_array_metadata_v3( + {**V3_ARRAY, "node_type": "grid", "fill_value": 300} + ) + + +def test_error_invalid_cannot_have_an_empty_report() -> None: + with pytest.raises(ValueError, match="at least one"): + Invalid(()) + + +def test_public_result_type_is_runtime_subscriptable() -> None: + assert get_args(ValidationResult[int]) == (Valid[int], Invalid) + + +def test_public_check_annotations_resolve_at_runtime() -> None: + hints = get_type_hints(check_array_metadata_v3) + assert get_args(hints["return"])[1] is Invalid + + +def test_discriminant_narrows_both_ways() -> None: + # The point of the union: `valid` selects which member is readable. + good = check_array_metadata_v3(V3_ARRAY) + if good.valid: + assert good.document["zarr_format"] == 3 + else: # pragma: no cover - the fixture is valid + pytest.fail("expected a Valid result") + + bad = check_array_metadata_v3({**V3_ARRAY, "fill_value": 300}) + if bad.valid: # pragma: no cover - the fixture is invalid + pytest.fail("expected an Invalid result") + else: + assert any(problem.loc == ("fill_value",) for problem in bad.problems) diff --git a/packages/zarr-metadata/tests/rules/test_rule_properties.py b/packages/zarr-metadata/tests/rules/test_rule_properties.py new file mode 100644 index 0000000000..12353cb709 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_rule_properties.py @@ -0,0 +1,171 @@ +"""Property tests for composition-rule boundaries and API agreement.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr_metadata.builder import create_zarr_v3_array_metadata_json +from zarr_metadata.model import MetadataValidationError +from zarr_metadata.rules import ( + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata.model import ValidationProblem + + +JSON_SCALARS = st.none() | st.booleans() | st.integers() | st.floats(allow_nan=False) | st.text() +JSON_VALUES = st.recursive( + JSON_SCALARS, + lambda children: ( + st.lists(children, max_size=4) | st.dictionaries(st.text(max_size=12), children, max_size=4) + ), + max_leaves=20, +) + + +DOCUMENT_VALIDATORS = ( + validate_array_metadata_v2, + validate_array_metadata_v3, + validate_group_metadata_v2, + validate_group_metadata_v3, +) + + +@pytest.mark.parametrize("validator", DOCUMENT_VALIDATORS, ids=lambda validator: validator.__name__) +@given(JSON_VALUES) +def test_document_validators_are_total_for_arbitrary_json( + validator: Callable[[object], tuple[ValidationProblem, ...]], value: object +) -> None: + """Untrusted JSON always produces a verdict; it never crashes the validator.""" + assert isinstance(validator(value), tuple) + + +@given( + extent=st.integers(min_value=0, max_value=200), + chunk_shapes=st.lists(st.integers(min_value=1, max_value=50), min_size=1, max_size=8), +) +def test_rectilinear_explicit_chunks_may_overflow_extent( + extent: int, chunk_shapes: list[int] +) -> None: + """The final explicit chunk may extend past the array boundary.""" + if sum(chunk_shapes) < extent: + return + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (extent,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (tuple(chunk_shapes),)}, + }, + "chunk_key_encoding": "default", + "codecs": ("bytes",), + } + assert validate_array_metadata_v3(document) == () + + +@given(depth=st.integers(min_value=1, max_value=5), leaf_fill=st.integers(0, 255)) +def test_nested_struct_fill_values_are_checked_recursively(depth: int, leaf_fill: int) -> None: + data_type: object = "uint8" + fill_value: object = leaf_fill + path: list[str] = [] + for level in range(depth): + name = f"level_{level}" + data_type = { + "name": "struct", + "configuration": {"fields": ({"name": name, "data_type": data_type},)}, + } + fill_value = {name: fill_value} + path.insert(0, name) + + document: dict[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "data_type": data_type, + "fill_value": fill_value, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), + } + assert validate_array_metadata_v3(document) == () + + invalid: object = 256 + for name in reversed(path): + invalid = {name: invalid} + document["fill_value"] = invalid + problems = validate_array_metadata_v3(document) + assert any(problem.loc == ("fill_value", *path) for problem in problems) + + +@given( + exponents=st.lists(st.integers(min_value=0, max_value=6), min_size=1, max_size=4, unique=True) +) +def test_nested_sharding_pipelines_accept_divisible_inner_chunks(exponents: list[int]) -> None: + """Every generated nesting level is checked against the level enclosing it.""" + inner_shapes = [2**exponent for exponent in sorted(exponents, reverse=True)] + codecs: tuple[object, ...] = ("bytes",) + for inner_shape in reversed(inner_shapes): + codecs = ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (inner_shape,), + "codecs": codecs, + "index_codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + "crc32c", + ), + }, + }, + ) + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (64,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (64,)}}, + "chunk_key_encoding": "default", + "codecs": codecs, + } + assert validate_array_metadata_v3(document) == () + + +@given( + data_type=st.sampled_from(("int8", "uint8", "int16", "uint16", "int32", "uint32")), + fill_value=st.integers(min_value=-(2**40), max_value=2**40), +) +def test_validator_and_factory_agree(data_type: str, fill_value: int) -> None: + document: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": data_type, + "fill_value": fill_value, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,)}}, + "chunk_key_encoding": "default", + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + } + validator_accepts = validate_array_metadata_v3(document) == () + + try: + create_zarr_v3_array_metadata_json(**document) # type: ignore[arg-type] + except MetadataValidationError: + factory_accepts = False + else: + factory_accepts = True + + assert validator_accepts == factory_accepts diff --git a/packages/zarr-metadata/tests/rules/test_spec_propagation.py b/packages/zarr-metadata/tests/rules/test_spec_propagation.py new file mode 100644 index 0000000000..e61733e8fa --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_spec_propagation.py @@ -0,0 +1,160 @@ +"""Tests for array-spec propagation through a codec chain. + +The property under test: every codec is judged against the array it +*receives*, which is the document's chunk only for the first codec in +the chain. Anything that transforms the array — a transpose, a cast, a +shard — changes what the next codec sees. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from zarr_metadata.rules import validate_array_metadata_v3 +from zarr_metadata.rules._spec import ( + NOTHING_KNOWN, + ArraySpec, + propagate, + transitions_registered, +) +from zarr_metadata.v3.codec.kind import ARRAY_ARRAY_CODEC_NAMES + +if TYPE_CHECKING: + from collections.abc import Mapping + + +def _doc(codecs: tuple[object, ...], chunk: tuple[int, ...] = (6, 4)) -> Mapping[str, object]: + return { + "zarr_format": 3, + "node_type": "array", + "shape": (12, 8), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": chunk}}, + "chunk_key_encoding": "default", + "codecs": codecs, + } + + +def _transpose(*order: int) -> Mapping[str, object]: + return {"name": "transpose", "configuration": {"order": order}} + + +def _shard( + inner: tuple[int, ...], + codecs: tuple[object, ...] = ("bytes",), + index_codecs: tuple[object, ...] = ({"name": "bytes", "configuration": {"endian": "little"}},), +) -> Mapping[str, object]: + return { + "name": "sharding_indexed", + "configuration": {"chunk_shape": inner, "codecs": codecs, "index_codecs": index_codecs}, + } + + +# (document, expected verdict). The chunk is (6, 4); a transpose (1, 0) in +# front of a shard means the shard receives (4, 6), and the verdict must +# follow the transposed shape, not the grid. +CASES: dict[str, tuple[Mapping[str, object], bool]] = { + "shard-alone-divides": (_doc((_shard((3, 2)),)), True), + "shard-alone-does-not-divide": (_doc((_shard((4, 3)),)), False), + # Regression: before propagation these two verdicts were reversed. + "transpose-then-shard-divides-transposed": (_doc((_transpose(1, 0), _shard((2, 3)))), True), + "transpose-then-shard-does-not-divide-transposed": ( + _doc((_transpose(1, 0), _shard((3, 2)))), + False, + ), + "two-transposes-cancel": (_doc((_transpose(1, 0), _transpose(1, 0), _shard((3, 2)))), True), + # Inside a shard the incoming array is the inner chunk, so a nested + # transpose is judged against the inner chunk's rank, and a nested + # shard against the transposed inner chunk. + "nested-transpose-matches-inner-rank": ( + _doc((_shard((3, 2), codecs=(_transpose(1, 0), "bytes")),)), + True, + ), + "nested-shard-follows-nested-transpose": ( + # inner chunk (3, 2) transposed -> (2, 3); nested shard (2, 1) divides it. + _doc((_shard((3, 2), codecs=(_transpose(1, 0), _shard((2, 1)))),)), + True, + ), + "nested-shard-violates-transposed-inner": ( + # inner chunk (3, 2) transposed -> (2, 3); nested shard (3, 1): 3 does not divide 2. + _doc((_shard((3, 2), codecs=(_transpose(1, 0), _shard((3, 1)))),)), + False, + ), + # An unknown codec might change the shape, so downstream geometry + # declines rather than guessing — an otherwise-invalid shard passes. + "unknown-codec-stops-propagation": (_doc(({"name": "zfpy"}, _shard((4, 3)))), True), + # The shard index is a uint64 array, so a bytes codec inside + # `index_codecs` needs an endianness like any multi-byte encoding. + "index-codecs-bare-bytes-needs-endian": ( + _doc((_shard((3, 2), index_codecs=("bytes", "crc32c")),)), + False, + ), +} + + +@pytest.mark.parametrize(("doc", "valid"), CASES.values(), ids=list(CASES)) +def test_verdict_follows_the_incoming_array(doc: Mapping[str, object], valid: bool) -> None: + problems = validate_array_metadata_v3(doc) + assert (len(problems) == 0) is valid, [str(p) for p in problems] + + +def test_error_locates_the_offending_shard() -> None: + problems = validate_array_metadata_v3(_doc((_transpose(1, 0), _shard((3, 2))))) + assert [(p.loc, p.kind) for p in problems] == [ + (("codecs", 1, "configuration", "chunk_shape", 0), "invalid_value") + ] + + +def test_propagate_yields_incoming_spec_per_codec() -> None: + from zarr_metadata.rules._registry import entity_configuration + from zarr_metadata.v3._extension_points import CODECS + + chain = (_transpose(1, 0), "bytes", "crc32c") + start = ArraySpec((6, 4), "uint8") + seen = list(propagate(chain, start, lambda c: entity_configuration(CODECS, c))) + incoming = [spec for _, _, spec in seen] + assert incoming[0] == ArraySpec((6, 4), "uint8") # transpose receives the chunk + assert incoming[1] == ArraySpec((4, 6), "uint8") # bytes receives the transposed chunk + # past array->bytes: no array, so no shape; the type carries through + assert incoming[2] == ArraySpec(None, "uint8") + + +def test_cast_value_changes_the_downstream_data_type() -> None: + from zarr_metadata.rules._registry import entity_configuration + from zarr_metadata.v3._extension_points import CODECS + + chain = ({"name": "cast_value", "configuration": {"data_type": "float32"}}, "bytes") + start = ArraySpec((6, 4), "uint8") + seen = list(propagate(chain, start, lambda c: entity_configuration(CODECS, c))) + assert seen[1][2] == ArraySpec((6, 4), "float32") + + +def test_unknown_codec_yields_nothing_known() -> None: + from zarr_metadata.rules._registry import entity_configuration + from zarr_metadata.v3._extension_points import CODECS + + start = ArraySpec((6, 4), "uint8") + seen = list( + propagate(({"name": "zfpy"}, "bytes"), start, lambda c: entity_configuration(CODECS, c)) + ) + assert seen[1][2] is NOTHING_KNOWN + + +def test_every_array_array_codec_registers_a_transition() -> None: + # A modelled array->array codec with no transition is treated as + # unknown and stops propagation — safe, but silently weaker than + # intended. Make it a decision, not an omission. + assert set(ARRAY_ARRAY_CODEC_NAMES) <= transitions_registered() | {"scale_offset"} + + +def test_error_transition_for_a_non_array_array_codec() -> None: + from zarr_metadata.rules._spec import spec_transition + + with pytest.raises(ValueError, match="only array->array codecs"): + + @spec_transition("gzip") + def _nope(configuration: object, incoming: ArraySpec) -> ArraySpec: # pragma: no cover + return incoming diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py new file mode 100644 index 0000000000..e3403726d5 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -0,0 +1,553 @@ +"""Tests for the v3 array and group composition rules added with the +rules-layer promotion: chunk grid values/geometry, transpose orders, +sharding pipelines/geometry, and consolidated-entry recursion.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from zarr_metadata.model import MetadataValidationError +from zarr_metadata.rules import validate_array_metadata_v3, validate_group_metadata_v3 + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import ZarrV3ArrayMetadataJSON + +BASE: ZarrV3ArrayMetadataJSON = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), +} + + +# The shard index is a uint64 array, so its bytes codec needs an endianness. +_INDEX_BYTES: Mapping[str, object] = {"name": "bytes", "configuration": {"endian": "little"}} + + +def _shard(**overrides: object) -> Mapping[str, object]: + """A sharding codec entry; overrides may be deliberately malformed.""" + configuration: dict[str, object] = { + "chunk_shape": (2, 2), + "codecs": ("bytes",), + "index_codecs": (_INDEX_BYTES, "crc32c"), + } + configuration.update(overrides) + return {"name": "sharding_indexed", "configuration": configuration} + + +# Documents that must be fully valid: the rules judge geometry and values +# without rejecting legitimate spellings of the same constructs. +VALID_CASES: dict[str, Mapping[str, object]] = { + "regular": BASE, + "rectilinear-explicit-sums": { + **BASE, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((2, 2), (1, 3))}, + }, + }, + "rectilinear-explicit-overflow": { + **BASE, + "shape": (6,), + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((4, 4, 4),)}, + }, + }, + "rectilinear-rle-and-uniform": { + **BASE, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (((2, 2),), 4)}, + }, + }, + "transpose": { + **BASE, + "codecs": ({"name": "transpose", "configuration": {"order": (1, 0)}}, "bytes"), + }, + "sharding": {**BASE, "codecs": (_shard(),)}, + "nested-sharding": { + **BASE, + "codecs": ( + _shard( + codecs=( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (1, 2), + "codecs": ("bytes",), + "index_codecs": (_INDEX_BYTES,), + }, + }, + ) + ), + ), + }, + "nested-struct": { + **BASE, + "data_type": { + "name": "struct", + "configuration": { + "fields": ( + {"name": "id", "data_type": "uint8"}, + { + "name": "point", + "data_type": { + "name": "struct", + "configuration": {"fields": ({"name": "x", "data_type": "int16"},)}, + }, + }, + ) + }, + }, + "fill_value": {"id": 1, "point": {"x": -2}}, + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + }, + "unknown-grid-passes": { + **BASE, + "chunk_grid": {"name": "hilbert", "configuration": {"level": 3}}, + }, + "unknown-codec-inconclusive": {**BASE, "codecs": ({"name": "zfpy"}, "bytes")}, +} + + +@pytest.mark.parametrize("doc", VALID_CASES.values(), ids=list(VALID_CASES)) +def test_valid_documents(doc: Mapping[str, object]) -> None: + assert validate_array_metadata_v3(doc) == () + + +def _sole_problem(doc: Mapping[str, object]) -> tuple[tuple[str | int, ...], str]: + problems = validate_array_metadata_v3(doc) + assert len(problems) == 1, [p.message for p in problems] + return problems[0].loc, problems[0].message + + +def test_error_regular_chunk_extent_zero() -> None: + loc, message = _sole_problem( + {**BASE, "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (0, 2)}}} + ) + assert loc == ("chunk_grid", "configuration", "chunk_shape", 0) + assert "positive chunk extent" in message + + +def test_error_rectilinear_rank_mismatch() -> None: + loc, message = _sole_problem( + { + **BASE, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((2, 2),)}, + }, + } + ) + assert loc == ("chunk_grid", "configuration", "chunk_shapes") + assert "2 dimensions" in message + + +def test_error_rectilinear_sum_mismatch() -> None: + loc, message = _sole_problem( + { + **BASE, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((3,), (2, 2))}, + }, + } + ) + assert loc == ("chunk_grid", "configuration", "chunk_shapes", 0) + assert "sum to 3" in message + + +def test_error_rectilinear_nonpositive_rle() -> None: + problems = validate_array_metadata_v3( + { + **BASE, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (((0, 4),), 4)}, + }, + } + ) + assert any("positive [size, count] pair" in p.message for p in problems) + + +def test_error_transpose_not_a_permutation() -> None: + loc, message = _sole_problem( + {**BASE, "codecs": ({"name": "transpose", "configuration": {"order": (5, 5)}}, "bytes")} + ) + assert loc == ("codecs", 0, "configuration", "order") + assert "permutation" in message + + +def test_error_transpose_rank_mismatch() -> None: + loc, message = _sole_problem( + {**BASE, "codecs": ({"name": "transpose", "configuration": {"order": (2, 0, 1)}}, "bytes")} + ) + assert loc == ("codecs", 0, "configuration", "order") + assert "incoming array has 2 dimensions" in message + + +def test_error_sharding_inner_pipeline_order() -> None: + loc, _ = _sole_problem({**BASE, "codecs": (_shard(codecs=("crc32c", "bytes")),)}) + assert loc == ("codecs", 0, "configuration", "codecs", 1) + + +def test_error_sharding_inner_no_array_bytes() -> None: + loc, message = _sole_problem({**BASE, "codecs": (_shard(codecs=("crc32c",)),)}) + assert loc == ("codecs", 0, "configuration", "codecs") + assert "no array->bytes codec" in message + + +def test_error_sharding_index_codecs_no_array_bytes() -> None: + loc, message = _sole_problem({**BASE, "codecs": (_shard(index_codecs=("crc32c",)),)}) + assert loc == ("codecs", 0, "configuration", "index_codecs") + assert "no array->bytes codec" in message + + +def test_error_sharding_index_codecs_are_variable_sized() -> None: + loc, message = _sole_problem( + { + **BASE, + "codecs": ( + _shard( + index_codecs=( + _INDEX_BYTES, + {"name": "gzip", "configuration": {"level": 1}}, + ) + ), + ), + } + ) + assert loc == ("codecs", 0, "configuration", "index_codecs", 1) + assert "fixed-size" in message + + +def test_error_sharding_rank_mismatch() -> None: + loc, message = _sole_problem({**BASE, "codecs": (_shard(chunk_shape=(2,)),)}) + assert loc == ("codecs", 0, "configuration", "chunk_shape") + assert "incoming array has 2 dimensions" in message + + +def test_error_sharding_not_divisible() -> None: + loc, message = _sole_problem( + { + **BASE, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (4, 4)}}, + "codecs": (_shard(chunk_shape=(3, 2)),), + } + ) + assert loc == ("codecs", 0, "configuration", "chunk_shape", 0) + assert "does not evenly divide" in message + + +def test_error_nested_sharding_not_divisible() -> None: + inner: Mapping[str, object] = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (2, 3), + "codecs": ("bytes",), + "index_codecs": (_INDEX_BYTES,), + }, + } + loc, message = _sole_problem( + { + **BASE, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (4, 4)}}, + "codecs": (_shard(codecs=(inner,)),), + } + ) + assert loc == ("codecs", 0, "configuration", "codecs", 0, "configuration", "chunk_shape", 1) + assert "does not evenly divide" in message + + +def test_error_sharding_inner_chunk_extent_zero() -> None: + problems = validate_array_metadata_v3({**BASE, "codecs": (_shard(chunk_shape=(0, 2)),)}) + assert any( + p.loc == ("codecs", 0, "configuration", "chunk_shape", 0) + and "positive chunk extent" in p.message + for p in problems + ) + + +def test_error_bytes_requires_endian_for_multibyte_data() -> None: + loc, message = _sole_problem({**BASE, "data_type": "int32", "codecs": ("bytes",)}) + assert loc == ("codecs", 0, "configuration", "endian") + assert "required" in message + + +def test_error_bytes_rejects_variable_length_data_type() -> None: + loc, message = _sole_problem( + {**BASE, "data_type": "string", "fill_value": "", "codecs": ("bytes",)} + ) + assert loc == ("codecs", 0, "configuration") + assert "not compatible" in message + + +def test_error_struct_fields_are_empty() -> None: + doc = { + **BASE, + "data_type": {"name": "struct", "configuration": {"fields": ()}}, + "fill_value": {}, + } + loc, message = _sole_problem(doc) + assert loc == ("data_type", "configuration", "fields") + assert "at least one" in message + + +def test_error_struct_field_is_variable_length() -> None: + doc = { + **BASE, + "data_type": { + "name": "struct", + "configuration": {"fields": ({"name": "label", "data_type": "string"},)}, + }, + "fill_value": {"label": ""}, + } + problems = validate_array_metadata_v3(doc) + assert any( + problem.loc == ("data_type", "configuration", "fields", 0, "data_type") + and "fixed-size" in problem.message + for problem in problems + ) + + +def test_error_struct_fill_is_missing_field() -> None: + doc = { + **BASE, + "data_type": { + "name": "struct", + "configuration": {"fields": ({"name": "x", "data_type": "uint8"},)}, + }, + "fill_value": {}, + } + loc, message = _sole_problem(doc) + assert loc == ("fill_value", "x") + assert "missing" in message + + +def test_error_struct_fill_field_is_invalid() -> None: + doc = { + **BASE, + "data_type": { + "name": "struct", + "configuration": {"fields": ({"name": "x", "data_type": "uint8"},)}, + }, + "fill_value": {"x": 300}, + } + loc, message = _sole_problem(doc) + assert loc == ("fill_value", "x") + assert "[0, 255]" in message + + +def test_error_gzip_level_is_out_of_range() -> None: + doc = { + **BASE, + "codecs": ("bytes", {"name": "gzip", "configuration": {"level": 99}}), + } + loc, message = _sole_problem(doc) + assert loc == ("codecs", 1, "configuration", "level") + assert "[0, 9]" in message + + +@pytest.mark.parametrize("data_type_name", ["numpy.datetime64", "numpy.timedelta64"]) +def test_error_numpy_time_scale_factor_is_out_of_range(data_type_name: str) -> None: + doc = { + **BASE, + "data_type": { + "name": data_type_name, + "configuration": {"unit": "ns", "scale_factor": 0}, + }, + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + } + loc, message = _sole_problem(doc) + assert loc == ("data_type", "configuration", "scale_factor") + assert "[1, 2147483647]" in message + + +@pytest.mark.parametrize("data_type_name", ["numpy.datetime64", "numpy.timedelta64"]) +def test_error_numpy_time_fill_is_out_of_range(data_type_name: str) -> None: + doc = { + **BASE, + "data_type": { + "name": data_type_name, + "configuration": {"unit": "ns", "scale_factor": 1}, + }, + "fill_value": 2**80, + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + } + loc, message = _sole_problem(doc) + assert loc == ("fill_value",) + assert "64-bit" in message + + +def test_error_consolidated_child_violates_array_rules() -> None: + doc: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {**BASE, "fill_value": 300}}, + }, + } + problems = validate_group_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("consolidated_metadata", "metadata", "a", "fill_value"), "invalid_value") + ] + + +def test_error_consolidated_nested_group_recursion() -> None: + child_group: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"b": {**BASE, "fill_value": 300}}, + }, + } + doc: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"g": child_group}, + }, + } + problems = validate_group_metadata_v3(doc) + assert [p.loc for p in problems] == [ + ( + "consolidated_metadata", + "metadata", + "g", + "consolidated_metadata", + "metadata", + "b", + "fill_value", + ) + ] + + +def test_error_group_parse_raises() -> None: + from zarr_metadata.rules import parse_group_metadata_v3 + + with pytest.raises(MetadataValidationError, match="fill_value invalid"): + parse_group_metadata_v3( + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": {"a": {**BASE, "fill_value": 300}}, + }, + } + ) + + +# -- unknown configuration members -------------------------------------------- +# +# The v3 spec does not say whether an extension's `configuration` is closed +# (zarr-developers/zarr-specs#270, open since 2023). This package takes the +# strict reading, matching most registered extension schemas and most other +# implementations — but reports it as its own `unknown_key` kind, and never +# lets it mask a real finding about the same entity. + + +def test_unknown_configuration_member_has_its_own_kind() -> None: + doc = { + **BASE, + "codecs": ({"name": "bytes", "configuration": {"endian": "little", "hint": 1}},), + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "unknown_key")] + + +def test_error_known_data_type_has_invalid_configuration() -> None: + doc = { + **BASE, + "data_type": { + "name": "numpy.datetime64", + "configuration": {"unit": "banana", "scale_factor": 1}, + }, + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("data_type", "configuration", "unit"), "invalid_value") + ] + + +def test_error_known_chunk_key_encoding_has_invalid_configuration() -> None: + doc = { + **BASE, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "!"}}, + } + problems = validate_array_metadata_v3(doc) + assert [(p.loc, p.kind) for p in problems] == [ + (("chunk_key_encoding", "configuration", "separator"), "invalid_value") + ] + + +def test_unknown_member_does_not_mask_a_codec_rule() -> None: + # Regression: an unrecognized member used to make the whole entity + # uninterpretable, silently suppressing every other rule about it — so a + # cosmetic extra key hid a genuine permutation error. + doc = { + **BASE, + "codecs": ({"name": "transpose", "configuration": {"order": (5, 5), "hint": 1}}, "bytes"), + } + kinds = {(p.loc, p.kind) for p in validate_array_metadata_v3(doc)} + assert (("codecs", 0, "configuration"), "unknown_key") in kinds + assert (("codecs", 0, "configuration", "order"), "invalid_value") in kinds + + +def test_unknown_member_does_not_mask_a_chunk_grid_rule() -> None: + doc = { + **BASE, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,), "hint": 1}}, + } + kinds = {(p.loc, p.kind) for p in validate_array_metadata_v3(doc)} + assert (("chunk_grid", "configuration"), "unknown_key") in kinds + assert (("chunk_grid", "configuration", "chunk_shape"), "invalid_value") in kinds + + +def test_unknown_member_survives_a_round_trip() -> None: + # Whatever the strict validator says, the package must never silently + # drop a member it does not model: a writer that knows more than we do + # must get its bytes back. (zarr-python's own chunk-grid path is lossy + # here; this asserts we are not.) + import json + + from zarr_metadata.model import ZarrV3ArrayMetadata + + raw = { + **BASE, + "shape": [4, 4], + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, 2]}}, + "codecs": [ + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "blocksize": 0, + "numThreads": 4, + }, + }, + "bytes", + ], + } + model = ZarrV3ArrayMetadata.from_json(json.loads(json.dumps(raw))) + emitted = model.to_json() + codec = emitted["codecs"][0] + assert codec["configuration"]["numThreads"] == 4 diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 6613aa394b..89be982958 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -281,23 +281,29 @@ def test_all_is_grouped_and_unique() -> None: "BloscShuffle", "CastOutOfRangeMode", "CastRoundingMode", + "CodecKind", "Endianness", "HexFloat16", "HexFloat32", "HexFloat64", "JSONValue", + "Invalid", "MetadataValidationError", "NumpyDatetime64", "NumpyTimeUnit", "NumpyTimedelta64", "ProblemKind", "RectilinearDimSpec", + "Rule", + "RuleCheck", "ScalarMap", "ScalarMapEntry", "ShardingIndexLocation", "Struct", "StructField", + "Valid", "ValidationProblem", + "ValidationResult", } ) @@ -409,7 +415,14 @@ def _literal_backed_constants() -> list[tuple[str, str, str]]: for const_name, value in vars(module).items(): if const_name.startswith("_") or not const_name.isupper(): continue - members = frozenset(value) if isinstance(value, tuple) else frozenset({value}) + if isinstance(value, tuple): + members = frozenset(value) + elif isinstance(value, str): + members = frozenset({value}) + else: + # Unhashable non-value constants (rule sets, factory + # registries) back no Literal type and carry no signal. + continue if not all(isinstance(m, str) for m in members): continue matches = [t for t, args in literals.items() if args == members] @@ -441,7 +454,14 @@ def _value_tied_constants() -> set[str]: for const_name, value in vars(module).items(): if const_name.startswith("_") or not const_name.isupper(): continue - members = frozenset(value) if isinstance(value, tuple) else frozenset({value}) + if isinstance(value, tuple): + members = frozenset(value) + elif isinstance(value, str): + members = frozenset({value}) + else: + # Unhashable non-value constants (rule sets, factory + # registries) back no Literal type and carry no signal. + continue if not all(isinstance(m, str) for m in members): continue if sum(1 for args in literals if args == members) > 1: diff --git a/packages/zarr-metadata/tests/test_registry_drift.py b/packages/zarr-metadata/tests/test_registry_drift.py new file mode 100644 index 0000000000..f21851b990 --- /dev/null +++ b/packages/zarr-metadata/tests/test_registry_drift.py @@ -0,0 +1,85 @@ +"""Drift tests tying the hand-written judgment registries to the package's +type modules: adding a codec, chunk grid, or data type module without +registering it in the corresponding judgment surface must fail a test +rather than silently weaken validation (an unregistered codec, for +example, would suppress the exactly-one-array->bytes check for every +pipeline containing it).""" + +from __future__ import annotations + +import importlib +import pkgutil + +import zarr_metadata.v3.chunk_grid +import zarr_metadata.v3.chunk_key_encoding +import zarr_metadata.v3.codec +import zarr_metadata.v3.data_type +from zarr_metadata.rules._v3_array import ( + _check_fill_for_dtype, # pyright: ignore[reportPrivateUsage] +) +from zarr_metadata.v3._extension_points import RAW_BYTES_FAMILY +from zarr_metadata.v3._shape import ( # pyright: ignore[reportPrivateUsage] + _CHUNK_GRID_SHAPES, + _CHUNK_KEY_ENCODING_SHAPES, + _CODEC_SHAPES, + _DATA_TYPE_SHAPES, +) +from zarr_metadata.v3.codec.kind import codec_kind_of_name + + +def _module_constants(package: object, suffix: str) -> set[str]: + """Values of `*` constants across a package's public modules.""" + names: set[str] = set() + for info in pkgutil.iter_modules(package.__path__): # type: ignore[attr-defined] + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{package.__name__}.{info.name}") # type: ignore[attr-defined] + names.update( + value + for attribute, value in vars(module).items() + if attribute.endswith(suffix) and isinstance(value, str) + ) + return names + + +def test_every_codec_module_is_kind_classified() -> None: + codec_names = _module_constants(zarr_metadata.v3.codec, "_CODEC_NAME") + assert codec_names, "constant scan found nothing — the naming convention moved?" + unclassified = {name for name in codec_names if codec_kind_of_name(name) is None} + assert not unclassified + + +def test_every_codec_module_has_a_shape_validator() -> None: + codec_names = _module_constants(zarr_metadata.v3.codec, "_CODEC_NAME") + assert codec_names == set(_CODEC_SHAPES) + + +def test_every_chunk_grid_module_has_a_shape_validator() -> None: + grid_names = _module_constants(zarr_metadata.v3.chunk_grid, "_CHUNK_GRID_NAME") + assert grid_names, "constant scan found nothing — the naming convention moved?" + assert grid_names == set(_CHUNK_GRID_SHAPES) + + +def test_every_chunk_key_encoding_module_has_a_shape_validator() -> None: + names = _module_constants(zarr_metadata.v3.chunk_key_encoding, "_CHUNK_KEY_ENCODING_NAME") + assert names, "constant scan found nothing — the naming convention moved?" + assert names == set(_CHUNK_KEY_ENCODING_SHAPES) + + +def test_every_data_type_module_has_a_shape_validator() -> None: + names = _module_constants(zarr_metadata.v3.data_type, "_DATA_TYPE_NAME") + assert names, "constant scan found nothing — the naming convention moved?" + assert names | {RAW_BYTES_FAMILY} == set(_DATA_TYPE_SHAPES) + + +def test_every_data_type_has_a_fill_value_branch() -> None: + # object() is a valid fill value for no data type this package + # defines, so a known name must produce a complaint; only genuinely + # unknown names may decline (extension openness). The parameterized + # r family has no name constant and is represented by "r8". + dtype_names = _module_constants(zarr_metadata.v3.data_type, "_DATA_TYPE_NAME") + assert dtype_names, "constant scan found nothing — the naming convention moved?" + unjudged = { + name for name in {*dtype_names, "r8"} if _check_fill_for_dtype(name, object()) is None + } + assert not unjudged diff --git a/packages/zarr-metadata/tests/v3/codec/test_kind.py b/packages/zarr-metadata/tests/v3/codec/test_kind.py new file mode 100644 index 0000000000..996d469035 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/codec/test_kind.py @@ -0,0 +1,26 @@ +"""Tests for codec kind classification.""" + +from __future__ import annotations + +import pytest + +from zarr_metadata.v3.codec.kind import codec_kind_of_name + +# (codec name, expected kind). Classification is by name alone. +CASES: dict[str, str | None] = { + "transpose": "array_array", + "cast_value": "array_array", + "scale_offset": "array_array", + "bytes": "array_bytes", + "sharding_indexed": "array_bytes", + "blosc": "bytes_bytes", + "crc32c": "bytes_bytes", + "gzip": "bytes_bytes", + "zstd": "bytes_bytes", + "lightspeed": None, +} + + +@pytest.mark.parametrize(("name", "kind"), CASES.items(), ids=list(CASES)) +def test_kind_of_name(name: str, kind: str | None) -> None: + assert codec_kind_of_name(name) == kind diff --git a/packages/zarr-metadata/tests/v3/test_extension_points.py b/packages/zarr-metadata/tests/v3/test_extension_points.py new file mode 100644 index 0000000000..7a539fe28d --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_extension_points.py @@ -0,0 +1,76 @@ +"""Tests for extension-point name canonicalization.""" + +from __future__ import annotations + +import pytest + +from zarr_metadata.rules import validate_array_metadata_v3 +from zarr_metadata.v3._extension_points import ( + CODECS, + DATA_TYPE, + RAW_BYTES_FAMILY, + canonical_name, +) + +# (field, name, expected canonical key) — identity everywhere except the +# parameterized raw-bytes family. +CANONICAL_CASES: dict[str, tuple[str, str, str]] = { + "plain-dtype": (DATA_TYPE, "uint8", "uint8"), + "dotted-dtype": (DATA_TYPE, "numpy.datetime64", "numpy.datetime64"), + "raw-8": (DATA_TYPE, "r8", RAW_BYTES_FAMILY), + "raw-24": (DATA_TYPE, "r24", RAW_BYTES_FAMILY), + # Malformed members canonicalize into the family too: a misspelling of + # something we model must be reported as such, not pass as an unknown + # third-party extension. + "raw-not-multiple-of-8": (DATA_TYPE, "r12", RAW_BYTES_FAMILY), + "raw-zero": (DATA_TYPE, "r0", RAW_BYTES_FAMILY), + # Canonicalization is field-aware: the r family is a data type. + "raw-shaped-codec-name": (CODECS, "r8", "r8"), + "codec": (CODECS, "blosc", "blosc"), + "unknown": (CODECS, "zfpy", "zfpy"), +} + + +@pytest.mark.parametrize( + ("field", "name", "expected"), CANONICAL_CASES.values(), ids=list(CANONICAL_CASES) +) +def test_canonical_name(field: str, name: str, expected: str) -> None: + assert canonical_name(field, name) == expected # type: ignore[arg-type] + + +def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: + # Zarr identifiers are registry-allocated. A private codec named + # `bytes` has left the compatibility contract, and saying so is the + # correct answer rather than a limitation, so nothing here defends + # against collisions. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ({"name": "bytes", "configuration": {"width": 7}},), + } + problems = validate_array_metadata_v3(document) + assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "unknown_key")] + + +def test_forging_the_family_sentinel_cannot_change_a_verdict() -> None: + # A literal "r" data type mislabels nothing: the rules layer matches + # the family through the name pattern, not through the table key, so + # no validation verdict depends on the sentinel being unforgeable. + assert canonical_name(DATA_TYPE, RAW_BYTES_FAMILY) == RAW_BYTES_FAMILY + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": RAW_BYTES_FAMILY, + "fill_value": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), + } + # Unjudged as an unknown data type, exactly as any unmodelled name is. + assert validate_array_metadata_v3(document) == () diff --git a/packages/zarr-metadata/tests/v3/test_shape_properties.py b/packages/zarr-metadata/tests/v3/test_shape_properties.py new file mode 100644 index 0000000000..1bcf8a3bf2 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_shape_properties.py @@ -0,0 +1,51 @@ +"""Generative invariants for raw-bytes name canonicalization. + +Scoped to canonicalization deliberately. Two earlier tests here asserted +that a shape verdict exists exactly when `(field, canonical_name(...))` +is in `modelled_entities()` — but both sides were computed from +`_ENTITY_SHAPES` through the same call, so they restated the lookup +rather than testing it, and could not fail. Worse, they could not catch +the bug class they named (a lookup passing the wrong field), because both +sides used the same field. `tests/rules/test_registry.py` covers that +with real assertions. + +Canonicalization is a genuine fit for generative testing: the family is +unbounded, so an example-based test can only sample it. +""" + +from __future__ import annotations + +from hypothesis import given +from hypothesis import strategies as st + +from zarr_metadata.v3._extension_points import ( + CHUNK_GRID, + CODECS, + DATA_TYPE, + RAW_BYTES_FAMILY, + canonical_name, +) + + +@given(width=st.integers(min_value=0, max_value=2**32)) +def test_every_numeric_r_spelling_folds_to_one_key(width: int) -> None: + # Including malformed widths (0, 12, anything not a multiple of 8): + # canonicalization is by grammar shape, not validity, so a misspelled + # member of a family we model is reported as a misspelling rather than + # passing as an unknown third-party extension. + assert canonical_name(DATA_TYPE, f"r{width}") == RAW_BYTES_FAMILY + + +@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from([CODECS, CHUNK_GRID])) +def test_r_shaped_names_are_identity_outside_data_types(width: int, field: str) -> None: + # The family belongs to `data_type`; a codec that happens to be named + # `r8` must not be folded into it. + name = f"r{width}" + assert canonical_name(field, name) == name # type: ignore[arg-type] + + +@given( + name=st.text(min_size=1).filter(lambda s: not (s.startswith("r") and s[1:].isdigit())), +) +def test_non_family_names_are_identity(name: str) -> None: + assert canonical_name(DATA_TYPE, name) == name From 793df0079daf20de63fae811f929a5ef9402d472 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 4 Sep 2026 09:46:35 +0200 Subject: [PATCH 002/107] chore(zarr-metadata): number changelog fragments for #318 Assisted-by: ClaudeCode:claude-fable-5-1 --- .../zarr-metadata/changes/{296.feature.3.md => 318.feature.3.md} | 0 .../zarr-metadata/changes/{296.feature.5.md => 318.feature.5.md} | 0 .../zarr-metadata/changes/{296.feature.6.md => 318.feature.6.md} | 0 packages/zarr-metadata/changes/{296.feature.md => 318.feature.md} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{296.feature.3.md => 318.feature.3.md} (100%) rename packages/zarr-metadata/changes/{296.feature.5.md => 318.feature.5.md} (100%) rename packages/zarr-metadata/changes/{296.feature.6.md => 318.feature.6.md} (100%) rename packages/zarr-metadata/changes/{296.feature.md => 318.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/296.feature.3.md b/packages/zarr-metadata/changes/318.feature.3.md similarity index 100% rename from packages/zarr-metadata/changes/296.feature.3.md rename to packages/zarr-metadata/changes/318.feature.3.md diff --git a/packages/zarr-metadata/changes/296.feature.5.md b/packages/zarr-metadata/changes/318.feature.5.md similarity index 100% rename from packages/zarr-metadata/changes/296.feature.5.md rename to packages/zarr-metadata/changes/318.feature.5.md diff --git a/packages/zarr-metadata/changes/296.feature.6.md b/packages/zarr-metadata/changes/318.feature.6.md similarity index 100% rename from packages/zarr-metadata/changes/296.feature.6.md rename to packages/zarr-metadata/changes/318.feature.6.md diff --git a/packages/zarr-metadata/changes/296.feature.md b/packages/zarr-metadata/changes/318.feature.md similarity index 100% rename from packages/zarr-metadata/changes/296.feature.md rename to packages/zarr-metadata/changes/318.feature.md From caf05c986cebca1dfff8d6224984d09745261547 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 12 Sep 2026 19:11:02 +0200 Subject: [PATCH 003/107] fix(zarr-metadata): validate nested data type composition Assisted-by: Codex:GPT-6 --- packages/zarr-metadata/changes/318.feature.md | 3 ++ .../rules/_entities/cast_value.py | 13 ++++- .../rules/_entities/struct_dtype.py | 22 +++++++- .../tests/rules/test_registry.py | 1 - .../tests/rules/test_rule_properties.py | 50 +++++++++++++++++++ 5 files changed, 86 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/changes/318.feature.md b/packages/zarr-metadata/changes/318.feature.md index 0e8c5b0e41..c91d3a9571 100644 --- a/packages/zarr-metadata/changes/318.feature.md +++ b/packages/zarr-metadata/changes/318.feature.md @@ -44,6 +44,9 @@ module there and changes nothing else. the transposed chunk, and a `bytes` codec behind a cast needs an endianness for the *target* type. `zarr_metadata.v3.codec.kind` sorts known codec names into the spec's three pipeline kinds. +- **Nested data types** in struct fields and cast targets recursively run + their registered composition rules, including time scale factors and + nested struct field constraints. - **Pydantic field types** for array and group documents now run the composition rules as well as structural validation. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py index 99f2964eb4..a14b6c4639 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py @@ -1,6 +1,6 @@ """Spec transition for the `cast_value` codec. -`cast_value` carries no composition rules of its own today, but it +`cast_value` validates its target data type and changes the data type everything downstream receives: a later rule that reads the type (the `bytes` codec's endianness requirement, for example) must judge against the configured target. @@ -16,15 +16,26 @@ from typing import TYPE_CHECKING, cast +from zarr_metadata.rules._registry import entity_rule, run_entity_rules from zarr_metadata.rules._spec import ArraySpec, spec_transition +from zarr_metadata.v3._extension_points import CODECS, DATA_TYPE from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME if TYPE_CHECKING: from collections.abc import Mapping + from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +@entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME) +def target_data_type_obeys_its_rules( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """A cast target obeys the same entity rules as a top-level data type.""" + return run_entity_rules(DATA_TYPE, configuration["data_type"], document, ("data_type",)) + + @spec_transition(CAST_VALUE_CODEC_NAME) def cast_data_type(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: """The outgoing type is the configured target.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py index f0739735d1..8be61b0361 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py @@ -12,7 +12,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.rules._engine import as_string_mapping -from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.rules._registry import entity_rule, run_entity_rules from zarr_metadata.v3._extension_points import DATA_TYPE from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME @@ -45,6 +45,26 @@ _VARIABLE_SIZE_NAMES = frozenset({"bytes", "string"}) +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +def field_data_types_obey_their_rules( + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec +) -> tuple[ValidationProblem, ...]: + """Apply every known data type's rules inside struct fields, recursively.""" + problems: list[ValidationProblem] = [] + for index, field in enumerate(cast("tuple[object, ...]", configuration["fields"])): + field_mapping = as_string_mapping(field) + if field_mapping is not None: + problems.extend( + run_entity_rules( + DATA_TYPE, + field_mapping.get("data_type"), + document, + ("fields", index, "data_type"), + ) + ) + return tuple(problems) + + def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str], ...]: """`(index, name)` for each field with a string name, else nothing. diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py index 2f748a70c1..6a65924839 100644 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -40,7 +40,6 @@ _RULE_FREE = frozenset( { (CODECS, "blosc"), - (CODECS, "cast_value"), (CODECS, "crc32c"), (CODECS, "scale_offset"), (CODECS, "zstd"), diff --git a/packages/zarr-metadata/tests/rules/test_rule_properties.py b/packages/zarr-metadata/tests/rules/test_rule_properties.py index 12353cb709..9f014c524a 100644 --- a/packages/zarr-metadata/tests/rules/test_rule_properties.py +++ b/packages/zarr-metadata/tests/rules/test_rule_properties.py @@ -169,3 +169,53 @@ def test_validator_and_factory_agree(data_type: str, fill_value: int) -> None: factory_accepts = True assert validator_accepts == factory_accepts + + +@pytest.mark.parametrize("position", ["data_type", "cast_value"]) +@pytest.mark.parametrize("name", ["numpy.datetime64", "numpy.timedelta64"]) +@given( + depth=st.integers(min_value=0, max_value=4), + scale=st.integers(min_value=-1, max_value=2**31), +) +def test_nested_time_data_types_obey_scale_factor_constraints( + position: str, name: str, depth: int, scale: int +) -> None: + """Embedding a known dtype must not bypass its composition rules.""" + data_type: object = { + "name": name, + "configuration": {"unit": "s", "scale_factor": scale}, + } + fill_value: object = 0 + nested_loc: tuple[str | int, ...] = ("configuration", "scale_factor") + for _ in range(depth): + data_type = { + "name": "struct", + "configuration": {"fields": ({"name": "value", "data_type": data_type},)}, + } + fill_value = {"value": fill_value} + nested_loc = ("configuration", "fields", 0, "data_type", *nested_loc) + document: dict[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "data_type": data_type, + "fill_value": fill_value, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "chunk_key_encoding": "default", + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + } + if position == "cast_value": + document["data_type"] = "uint8" + document["fill_value"] = 0 + document["codecs"] = ( + {"name": "cast_value", "configuration": {"data_type": data_type}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ) + loc = ("codecs", 0, "configuration", "data_type", *nested_loc) + else: + loc = ("data_type", *nested_loc) + problems = validate_array_metadata_v3(document) + if 1 <= scale < 2**31: + assert problems == () + else: + assert any(problem.loc == loc for problem in problems) From f2919baf846fa848aff4eb84cd78b11515365914 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 19 Sep 2026 18:54:14 +0200 Subject: [PATCH 004/107] fix(zarr-metadata): replace runtime asserts in the rules layer and create_* factories Upstream enabled ruff S101 for runtime code (#4363). The six parse-then-raise factories share a _parsed_or_raise helper that narrows the parsed document, and the rule-registration import uses importlib.import_module instead of an assert to keep it referenced. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/builder/_create.py | 41 ++++++++++--------- .../src/zarr_metadata/rules/_v3_array.py | 6 +-- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/_create.py b/packages/zarr-metadata/src/zarr_metadata/builder/_create.py index 58aac4c47b..0be7ee2821 100644 --- a/packages/zarr-metadata/src/zarr_metadata/builder/_create.py +++ b/packages/zarr-metadata/src/zarr_metadata/builder/_create.py @@ -12,7 +12,7 @@ import copy from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, TypeVar, cast from typing_extensions import Unpack @@ -92,6 +92,21 @@ def _raise_if_problems(problems: Sequence[ValidationProblem]) -> None: raise MetadataValidationError(problems) +_T = TypeVar("_T") + + +def _parsed_or_raise(parsed: _T | None, problems: Sequence[ValidationProblem]) -> _T: + """Return `parsed`, raising the collected problems if there are any. + + A parse that raised contributed its problems, so `parsed` is only `None` + when `problems` is non-empty; the final check guards that invariant. + """ + _raise_if_problems(problems) + if parsed is None: + raise RuntimeError("parser returned no document and reported no problems") + return parsed + + def _reject_attributes(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: """Problems for an `attributes` key in a strict on-disk v2 document. @@ -136,9 +151,7 @@ def create_zarr_v3_array_metadata_json( except MetadataValidationError as error: problems.extend(error.problems) problems.extend(run_rules(ZARR_V3_ARRAY_RULES, normalized)) - _raise_if_problems(problems) - assert parsed is not None - return parsed + return _parsed_or_raise(parsed, problems) def create_zarr_v3_group_metadata_json( @@ -163,9 +176,7 @@ def create_zarr_v3_group_metadata_json( except MetadataValidationError as error: problems.extend(error.problems) problems.extend(run_rules(ZARR_V3_GROUP_RULES, normalized)) - _raise_if_problems(problems) - assert parsed is not None - return parsed + return _parsed_or_raise(parsed, problems) def create_zarr_v3_consolidated_metadata_json( @@ -199,9 +210,7 @@ def create_zarr_v2_array_metadata_json( except MetadataValidationError as error: problems.extend(error.problems) problems.extend(run_rules(ZARR_V2_ARRAY_RULES, normalized)) - _raise_if_problems(problems) - assert parsed is not None - return parsed + return _parsed_or_raise(parsed, problems) def create_zarr_v2_group_metadata_json( @@ -218,9 +227,7 @@ def create_zarr_v2_group_metadata_json( parsed = parse_group_metadata_v2(_normalized(kwargs)) except MetadataValidationError as error: problems.extend(error.problems) - _raise_if_problems(problems) - assert parsed is not None - return parsed + return _parsed_or_raise(parsed, problems) def create_zarr_v2_zarray_json(**kwargs: Unpack[ZarrV2ZArrayJSON]) -> ZarrV2ZArrayJSON: @@ -239,9 +246,7 @@ def create_zarr_v2_zarray_json(**kwargs: Unpack[ZarrV2ZArrayJSON]) -> ZarrV2ZArr except MetadataValidationError as error: problems.extend(error.problems) problems.extend(run_rules(ZARR_V2_ARRAY_RULES, normalized)) - _raise_if_problems(problems) - assert parsed is not None - return cast("ZarrV2ZArrayJSON", parsed) + return cast("ZarrV2ZArrayJSON", _parsed_or_raise(parsed, problems)) def create_zarr_v2_zgroup_json(**kwargs: Unpack[ZarrV2ZGroupJSON]) -> ZarrV2ZGroupJSON: @@ -259,9 +264,7 @@ def create_zarr_v2_zgroup_json(**kwargs: Unpack[ZarrV2ZGroupJSON]) -> ZarrV2ZGro parsed = parse_group_metadata_v2(normalized) except MetadataValidationError as error: problems.extend(error.problems) - _raise_if_problems(problems) - assert parsed is not None - return cast("ZarrV2ZGroupJSON", parsed) + return cast("ZarrV2ZGroupJSON", _parsed_or_raise(parsed, problems)) def _validate_v2_consolidated_envelope( diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py index 1391707ddb..e012db51aa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py @@ -21,6 +21,7 @@ from __future__ import annotations +import importlib from collections.abc import Mapping from typing import TYPE_CHECKING, Final, cast @@ -406,11 +407,8 @@ def check_dimension_names_length(document: Mapping[str, object]) -> tuple[Valida def _rules() -> tuple[Rule, ...]: # Importing the entity package registers every entity's rules; done here # rather than at module import to keep the dependency one-directional. - import zarr_metadata.rules._entities as entity_rules_package + importlib.import_module("zarr_metadata.rules._entities") - # Imported for its registrations; referenced so the import cannot be - # pruned as unused by a checker or a well-meaning cleanup. - assert entity_rules_package is not None return document_rules(ZARR_V3_ARRAY) From 3f082ab2d5d3424b0bbbe58f6481e04bc242df15 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 19 Sep 2026 18:55:10 +0200 Subject: [PATCH 005/107] chore(zarr-metadata): number changelog fragments for #4379 Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/{318.feature.3.md => 4379.feature.3.md} | 0 .../zarr-metadata/changes/{318.feature.5.md => 4379.feature.5.md} | 0 .../zarr-metadata/changes/{318.feature.6.md => 4379.feature.6.md} | 0 .../zarr-metadata/changes/{318.feature.md => 4379.feature.md} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{318.feature.3.md => 4379.feature.3.md} (100%) rename packages/zarr-metadata/changes/{318.feature.5.md => 4379.feature.5.md} (100%) rename packages/zarr-metadata/changes/{318.feature.6.md => 4379.feature.6.md} (100%) rename packages/zarr-metadata/changes/{318.feature.md => 4379.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/318.feature.3.md b/packages/zarr-metadata/changes/4379.feature.3.md similarity index 100% rename from packages/zarr-metadata/changes/318.feature.3.md rename to packages/zarr-metadata/changes/4379.feature.3.md diff --git a/packages/zarr-metadata/changes/318.feature.5.md b/packages/zarr-metadata/changes/4379.feature.5.md similarity index 100% rename from packages/zarr-metadata/changes/318.feature.5.md rename to packages/zarr-metadata/changes/4379.feature.5.md diff --git a/packages/zarr-metadata/changes/318.feature.6.md b/packages/zarr-metadata/changes/4379.feature.6.md similarity index 100% rename from packages/zarr-metadata/changes/318.feature.6.md rename to packages/zarr-metadata/changes/4379.feature.6.md diff --git a/packages/zarr-metadata/changes/318.feature.md b/packages/zarr-metadata/changes/4379.feature.md similarity index 100% rename from packages/zarr-metadata/changes/318.feature.md rename to packages/zarr-metadata/changes/4379.feature.md From a63933ca1a9b097df6a7966ab86bdea8ab006530 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:09:34 +0200 Subject: [PATCH 006/107] refactor(zarr-metadata)!: drop the create_* builder from the rules PR The builder is a construction feature in a read-side PR, and its factories re-implemented the structure-plus-composition combination this PR already exposes as `rules.validate_*` / `rules.parse_*`. Held back for a later PR alongside the incremental builder. `test_validator_and_factory_agree` becomes `test_validator_and_parser_agree`: the same property against the front door the package keeps. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/README.md | 4 - .../zarr-metadata/changes/4379.feature.3.md | 14 - .../zarr-metadata/changes/4379.feature.md | 9 +- packages/zarr-metadata/docs/api/builder.md | 5 - packages/zarr-metadata/docs/api/index.md | 2 - packages/zarr-metadata/mkdocs.yml | 1 - .../src/zarr_metadata/builder/__init__.py | 32 -- .../src/zarr_metadata/builder/_create.py | 336 ------------------ .../zarr-metadata/tests/builder/__init__.py | 0 .../tests/builder/test_create.py | 254 ------------- .../tests/rules/test_rule_properties.py | 12 +- 11 files changed, 10 insertions(+), 659 deletions(-) delete mode 100644 packages/zarr-metadata/changes/4379.feature.3.md delete mode 100644 packages/zarr-metadata/docs/api/builder.md delete mode 100644 packages/zarr-metadata/src/zarr_metadata/builder/__init__.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/builder/_create.py delete mode 100644 packages/zarr-metadata/tests/builder/__init__.py delete mode 100644 packages/zarr-metadata/tests/builder/test_create.py diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 0a0aa89ec1..da0052a0c6 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -39,10 +39,6 @@ document = parse_array_metadata_v3(raw) # raises with every problem found metadata = ZarrV3ArrayMetadata.from_json(document) ``` -To construct a document, the `create_*` factories in `zarr_metadata.builder` -apply the same judgment to keyword arguments typed by the document's -`TypedDict`. - The optional Pydantic integration runs raw input through the rules layer and returns the same normalized model class: diff --git a/packages/zarr-metadata/changes/4379.feature.3.md b/packages/zarr-metadata/changes/4379.feature.3.md deleted file mode 100644 index f699ac068f..0000000000 --- a/packages/zarr-metadata/changes/4379.feature.3.md +++ /dev/null @@ -1,14 +0,0 @@ -Added `create_*` factories in `zarr_metadata.builder`, one per public -document TypedDict (`create_zarr_v3_array_metadata_json`, -`create_zarr_v3_group_metadata_json`, `create_zarr_v3_consolidated_metadata_json`, -`create_zarr_v2_array_metadata_json`, `create_zarr_v2_group_metadata_json`, -`create_zarr_v2_zarray_json`, `create_zarr_v2_zgroup_json`, -`create_zarr_v2_consolidated_metadata_json`), each taking -`**kwargs: Unpack[]`. Each factory copies and normalizes its -input, runs structural and composition validation, and raises one -`MetadataValidationError` containing all problems. The strict on-disk -`.zarray`/`.zgroup` factories reject `attributes` at runtime, and the v2 -consolidated factory validates each entry against the document shape its -path suffix selects. The open v3 array/group factories take an -`extensions=` mapping for extension fields (for type checkers without PEP -728 support) and reject names that shadow standard fields. diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index c91d3a9571..f8326ac6b1 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -1,9 +1,8 @@ Added `zarr_metadata.rules`: composition rules for full metadata -documents. The package now models metadata in three layers with one -contract each — `model` checks structure element by element, `rules` -judges composition across the document, and `builder` constructs while -applying both. Rules are registered where they are defined; rules about a -particular codec, chunk grid, or data type live with that entity under +documents. The package now models metadata in two layers with one +contract each — `model` checks structure element by element and `rules` +judges composition across the document. Rules are registered where they +are defined; rules about a particular codec, chunk grid, or data type live with that entity under `rules._entities` and are dispatched by name, so adding an entity adds a module there and changes nothing else. diff --git a/packages/zarr-metadata/docs/api/builder.md b/packages/zarr-metadata/docs/api/builder.md deleted file mode 100644 index 50bf9ac278..0000000000 --- a/packages/zarr-metadata/docs/api/builder.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: builder ---- - -::: zarr_metadata.builder diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 1a73ec3842..788cffece8 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -12,8 +12,6 @@ The package is organized to mirror the structure of the Zarr specifications: judgments over full documents (fill value vs. data type, codec pipeline ordering, chunk geometry), plus whole-document `validate`/`is`/`parse` trios combining structure and composition -- [`zarr_metadata.builder`](builder.md) — validated construction: - one-shot `create_*` factories, one per document type - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 225fb600e4..24b3e28240 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -17,7 +17,6 @@ nav: - api/index.md - ' zarr_metadata.model': api/model.md - ' zarr_metadata.rules': api/rules.md - - ' zarr_metadata.builder': api/builder.md - ' zarr_metadata.pydantic': api/pydantic.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py b/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py deleted file mode 100644 index 10dcb0dfe9..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Validated construction of Zarr metadata documents. - -`create_*` factories provide typed one-shot construction for every -document TypedDict: required keys and value types are checked statically -at literal-keyword call sites, and the runtime pass normalizes the input -and applies structural and composition validation, raising one -`MetadataValidationError` carrying every problem. - -Use `zarr_metadata.rules` to validate documents read from storage. -""" - -from zarr_metadata.builder._create import ( - create_zarr_v2_array_metadata_json, - create_zarr_v2_consolidated_metadata_json, - create_zarr_v2_group_metadata_json, - create_zarr_v2_zarray_json, - create_zarr_v2_zgroup_json, - create_zarr_v3_array_metadata_json, - create_zarr_v3_consolidated_metadata_json, - create_zarr_v3_group_metadata_json, -) - -__all__ = [ - "create_zarr_v2_array_metadata_json", - "create_zarr_v2_consolidated_metadata_json", - "create_zarr_v2_group_metadata_json", - "create_zarr_v2_zarray_json", - "create_zarr_v2_zgroup_json", - "create_zarr_v3_array_metadata_json", - "create_zarr_v3_consolidated_metadata_json", - "create_zarr_v3_group_metadata_json", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/_create.py b/packages/zarr-metadata/src/zarr_metadata/builder/_create.py deleted file mode 100644 index 0be7ee2821..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/builder/_create.py +++ /dev/null @@ -1,336 +0,0 @@ -"""One-shot factories for metadata document TypedDicts. - -Each factory copies and normalizes its input, then applies structural and -composition validation. Invalid input raises one -`MetadataValidationError`. V3 array and group factories accept extension -fields through `extensions=` for compatibility with type checkers that do -not support PEP 728. - -""" - -from __future__ import annotations - -import copy -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, TypeVar, cast - -from typing_extensions import Unpack - -from zarr_metadata.model._group import ZarrV2ConsolidatedMetadata -from zarr_metadata.model._validation import ( - ARRAY_METADATA_STANDARD_KEYS_V3, - GROUP_METADATA_STANDARD_KEYS_V3, - MetadataValidationError, - ValidationProblem, - arrays_to_tuples, - parse_array_metadata_v2, - parse_array_metadata_v3, - parse_group_metadata_v2, - parse_group_metadata_v3, - validate_consolidated_metadata_v3, -) -from zarr_metadata.rules import ( - ZARR_V2_ARRAY_RULES, - ZARR_V3_ARRAY_RULES, - ZARR_V3_GROUP_RULES, - run_rules, - validate_array_metadata_v2, - validate_group_metadata_v2, -) -from zarr_metadata.rules._v3_group import consolidated_entries_problems - -if TYPE_CHECKING: - from collections.abc import Set as AbstractSet - - from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON, ZarrV2ZArrayJSON - from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataJSON - from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2ZGroupJSON - from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON, ZarrV3ExtensionField - from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON - from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON - - -def _merged_with_extensions( - kwargs: Mapping[str, object], - extensions: Mapping[str, ZarrV3ExtensionField] | None, - standard_keys: AbstractSet[str], -) -> tuple[dict[str, object], list[ValidationProblem]]: - """`kwargs` plus `extensions`, refusing extension names that shadow standard keys. - - A colliding name is reported and *not* merged, so a later structural or - semantic pass judges the standard field the caller actually passed - rather than a value smuggled in through the extension hatch. - """ - document = dict(kwargs) - problems: list[ValidationProblem] = [] - for name, value in (extensions or {}).items(): - if name in standard_keys: - problems.append( - ValidationProblem( - (name,), - f"{name!r} is a standard metadata key; pass it as a keyword argument", - "invalid_value", - ) - ) - else: - document[name] = value - return document, problems - - -def _normalized(document: Mapping[str, object]) -> dict[str, object]: - """A deep copy of `document` with JSON arrays materialized as tuples. - - Deep-copying first means the returned document shares no mutable state - with the caller's arguments: mutating an input after the factory - returns cannot alter the validated result. - """ - return cast("dict[str, object]", arrays_to_tuples(copy.deepcopy(dict(document)))) - - -def _raise_if_problems(problems: Sequence[ValidationProblem]) -> None: - if len(problems) != 0: - raise MetadataValidationError(problems) - - -_T = TypeVar("_T") - - -def _parsed_or_raise(parsed: _T | None, problems: Sequence[ValidationProblem]) -> _T: - """Return `parsed`, raising the collected problems if there are any. - - A parse that raised contributed its problems, so `parsed` is only `None` - when `problems` is non-empty; the final check guards that invariant. - """ - _raise_if_problems(problems) - if parsed is None: - raise RuntimeError("parser returned no document and reported no problems") - return parsed - - -def _reject_attributes(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """Problems for an `attributes` key in a strict on-disk v2 document. - - The strict `.zarray` / `.zgroup` shapes exclude `attributes` (it lives - in the sibling `.zattrs` file). The signature enforces that statically - at keyword call sites; this is the runtime backstop for `**`-splatted - and untyped callers, without which the merged-form parser would accept - the key and the returned value would not be the type it claims. - """ - if "attributes" not in document: - return () - return ( - ValidationProblem( - ("attributes",), - "'attributes' is not part of the on-disk document (it belongs to the " - "sibling .zattrs file); use the merged-form factory instead", - "invalid_value", - ), - ) - - -def create_zarr_v3_array_metadata_json( - *, - extensions: Mapping[str, ZarrV3ExtensionField] | None = None, - **kwargs: Unpack[ZarrV3ArrayMetadataJSON], -) -> ZarrV3ArrayMetadataJSON: - """A validated v3 array metadata document (the `zarr.json` content for an array). - - Required keys are enforced statically by the signature; at runtime the - document is checked structurally (via the model layer's parser) and - semantically (via `ZARR_V3_ARRAY_RULES`), and every problem from both - passes is raised together in one `MetadataValidationError`. Extension - fields go in `extensions`; names that shadow standard keys are rejected. - """ - document, problems = _merged_with_extensions( - kwargs, extensions, ARRAY_METADATA_STANDARD_KEYS_V3 - ) - normalized = _normalized(document) - parsed: ZarrV3ArrayMetadataJSON | None = None - try: - parsed = parse_array_metadata_v3(normalized) - except MetadataValidationError as error: - problems.extend(error.problems) - problems.extend(run_rules(ZARR_V3_ARRAY_RULES, normalized)) - return _parsed_or_raise(parsed, problems) - - -def create_zarr_v3_group_metadata_json( - *, - extensions: Mapping[str, ZarrV3ExtensionField] | None = None, - **kwargs: Unpack[ZarrV3GroupMetadataJSON], -) -> ZarrV3GroupMetadataJSON: - """A validated v3 group metadata document (the `zarr.json` content for a group). - - Extension fields go in `extensions`; names that shadow standard keys - are rejected. The composition rules recurse into inline consolidated - metadata, so an embedded child document invalid under its own rules - is reported here, at its path. - """ - document, problems = _merged_with_extensions( - kwargs, extensions, GROUP_METADATA_STANDARD_KEYS_V3 - ) - normalized = _normalized(document) - parsed: ZarrV3GroupMetadataJSON | None = None - try: - parsed = parse_group_metadata_v3(normalized) - except MetadataValidationError as error: - problems.extend(error.problems) - problems.extend(run_rules(ZARR_V3_GROUP_RULES, normalized)) - return _parsed_or_raise(parsed, problems) - - -def create_zarr_v3_consolidated_metadata_json( - **kwargs: Unpack[ZarrV3ConsolidatedMetadataJSON], -) -> ZarrV3ConsolidatedMetadataJSON: - """A validated v3 inline consolidated metadata object. - - This is the value embedded in a v3 group document under the - `consolidated_metadata` key, not a store document of its own. - """ - normalized = _normalized(kwargs) - _raise_if_problems( - validate_consolidated_metadata_v3(normalized) + consolidated_entries_problems(normalized) - ) - return cast("ZarrV3ConsolidatedMetadataJSON", normalized) - - -def create_zarr_v2_array_metadata_json( - **kwargs: Unpack[ZarrV2ArrayMetadataJSON], -) -> ZarrV2ArrayMetadataJSON: - """A validated v2 array metadata document, in-memory merged form. - - Models `.zarray` plus the sibling `.zattrs` folded in as `attributes`. - For the strict on-disk `.zarray` shape use `create_zarr_v2_zarray_json`. - """ - normalized = _normalized(kwargs) - parsed: ZarrV2ArrayMetadataJSON | None = None - problems: list[ValidationProblem] = [] - try: - parsed = parse_array_metadata_v2(normalized) - except MetadataValidationError as error: - problems.extend(error.problems) - problems.extend(run_rules(ZARR_V2_ARRAY_RULES, normalized)) - return _parsed_or_raise(parsed, problems) - - -def create_zarr_v2_group_metadata_json( - **kwargs: Unpack[ZarrV2GroupMetadataJSON], -) -> ZarrV2GroupMetadataJSON: - """A validated v2 group metadata document, in-memory merged form. - - Models `.zgroup` plus the sibling `.zattrs` folded in as `attributes`. - For the strict on-disk `.zgroup` shape use `create_zarr_v2_zgroup_json`. - """ - parsed: ZarrV2GroupMetadataJSON | None = None - problems: list[ValidationProblem] = [] - try: - parsed = parse_group_metadata_v2(_normalized(kwargs)) - except MetadataValidationError as error: - problems.extend(error.problems) - return _parsed_or_raise(parsed, problems) - - -def create_zarr_v2_zarray_json(**kwargs: Unpack[ZarrV2ZArrayJSON]) -> ZarrV2ZArrayJSON: - """A validated on-disk `.zarray` document (strict form, no `attributes`). - - Structurally checked with the merged-form parser plus a runtime - rejection of `attributes`: the strict shape is the merged shape minus - `attributes`, and the runtime check holds for callers the signature's - static exclusion cannot see (`**`-splatted mappings, untyped code). - """ - normalized = _normalized(kwargs) - problems: list[ValidationProblem] = list(_reject_attributes(normalized)) - parsed: ZarrV2ArrayMetadataJSON | None = None - try: - parsed = parse_array_metadata_v2(normalized) - except MetadataValidationError as error: - problems.extend(error.problems) - problems.extend(run_rules(ZARR_V2_ARRAY_RULES, normalized)) - return cast("ZarrV2ZArrayJSON", _parsed_or_raise(parsed, problems)) - - -def create_zarr_v2_zgroup_json(**kwargs: Unpack[ZarrV2ZGroupJSON]) -> ZarrV2ZGroupJSON: - """A validated on-disk `.zgroup` document (strict form, no `attributes`). - - Structurally checked with the merged-form parser plus a runtime - rejection of `attributes`: the strict shape is the merged shape minus - `attributes`, and the runtime check holds for callers the signature's - static exclusion cannot see (`**`-splatted mappings, untyped code). - """ - normalized = _normalized(kwargs) - problems: list[ValidationProblem] = list(_reject_attributes(normalized)) - parsed: ZarrV2GroupMetadataJSON | None = None - try: - parsed = parse_group_metadata_v2(normalized) - except MetadataValidationError as error: - problems.extend(error.problems) - return cast("ZarrV2ZGroupJSON", _parsed_or_raise(parsed, problems)) - - -def _validate_v2_consolidated_envelope( - document: Mapping[str, object], -) -> tuple[ValidationProblem, ...]: - """Every reason `document` is not a `.zmetadata` envelope. - - The envelope itself is the model layer's judgment; on top of it, each - entry's path suffix selects the strict on-disk document shape that its - value must satisfy. - """ - try: - ZarrV2ConsolidatedMetadata.from_json(document) - except MetadataValidationError as error: - return error.problems - problems: list[ValidationProblem] = [] - for key, entry in cast("Mapping[str, object]", document["metadata"]).items(): - if not isinstance(entry, Mapping): - problems.append( - ValidationProblem(("metadata", key), "expected a JSON object", "invalid_type") - ) - continue - entry_mapping = cast("Mapping[str, object]", entry) - if key.endswith(".zarray"): - nested = validate_array_metadata_v2(entry_mapping) + _reject_attributes(entry_mapping) - elif key.endswith(".zgroup"): - nested = validate_group_metadata_v2(entry_mapping) + _reject_attributes(entry_mapping) - elif key.endswith(".zattrs"): - nested = () - else: - nested = ( - ValidationProblem( - (), - "expected a v2 metadata file suffix: .zarray, .zgroup, or .zattrs", - "invalid_value", - ), - ) - problems.extend( - ValidationProblem(("metadata", key, *found.loc), found.message, found.kind) - for found in nested - ) - return tuple(problems) - - -def create_zarr_v2_consolidated_metadata_json( - **kwargs: Unpack[ZarrV2ConsolidatedMetadataJSON], -) -> ZarrV2ConsolidatedMetadataJSON: - """A validated `.zmetadata` consolidated metadata document. - - The runtime pass checks the envelope and validates each nested value - against the strict document shape selected by its path suffix. This is - the runtime backstop for callers the signature's static enforcement - cannot see (`**`-splatted mappings, untyped code). - """ - normalized = _normalized(kwargs) - _raise_if_problems(_validate_v2_consolidated_envelope(normalized)) - return cast("ZarrV2ConsolidatedMetadataJSON", normalized) - - -__all__ = [ - "create_zarr_v2_array_metadata_json", - "create_zarr_v2_consolidated_metadata_json", - "create_zarr_v2_group_metadata_json", - "create_zarr_v2_zarray_json", - "create_zarr_v2_zgroup_json", - "create_zarr_v3_array_metadata_json", - "create_zarr_v3_consolidated_metadata_json", - "create_zarr_v3_group_metadata_json", -] diff --git a/packages/zarr-metadata/tests/builder/__init__.py b/packages/zarr-metadata/tests/builder/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/zarr-metadata/tests/builder/test_create.py b/packages/zarr-metadata/tests/builder/test_create.py deleted file mode 100644 index f40cd56206..0000000000 --- a/packages/zarr-metadata/tests/builder/test_create.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Tests for the `create_*` document factories.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest -import typing_extensions - -import zarr_metadata -import zarr_metadata.builder -from zarr_metadata.builder._create import ( - create_zarr_v2_array_metadata_json, - create_zarr_v2_consolidated_metadata_json, - create_zarr_v2_group_metadata_json, - create_zarr_v2_zarray_json, - create_zarr_v2_zgroup_json, - create_zarr_v3_array_metadata_json, - create_zarr_v3_consolidated_metadata_json, - create_zarr_v3_group_metadata_json, -) -from zarr_metadata.model import MetadataValidationError - -if TYPE_CHECKING: - from collections.abc import Callable, Mapping - - # Factories differ in the document type they return; these tests only - # ever compare the result as a mapping. - Factory = Callable[..., Mapping[str, object]] - -V3_ARRAY: dict[str, object] = { - "zarr_format": 3, - "node_type": "array", - "shape": (4, 4), - "data_type": "uint8", - "fill_value": 0, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, - "chunk_key_encoding": "default", - "codecs": ("bytes",), -} - -V2_ZARRAY: dict[str, object] = { - "zarr_format": 2, - "shape": (4,), - "chunks": (2,), - "dtype": " None: - assert factory(**kwargs) == expected - - -def test_output_shares_no_state_with_arguments() -> None: - grid: dict[str, object] = {"name": "regular", "configuration": {"chunk_shape": (2, 2)}} - document = create_zarr_v3_array_metadata_json(**{**V3_ARRAY, "chunk_grid": grid}) - grid["configuration"]["chunk_shape"] = (9, 9) # caller mutates after the fact - assert document["chunk_grid"]["configuration"]["chunk_shape"] == (2, 2) - - -# -- the package rule, enforced ---------------------------------------------- - -# Public TypedDicts ending in JSON that are field/helper shapes rather than -# documents. Closed by hand, like the naming-grammar vocabulary. -_HELPER_SHAPES = frozenset({"ZarrV3NamedConfigJSON"}) - -# Every public document TypedDict, mapped to its factory. Kept here rather -# than in the package: the mapping exists only so this test can hold the -# two sets equal. -_FACTORIES: dict[str, Factory] = { - "ZarrV2ArrayMetadataJSON": create_zarr_v2_array_metadata_json, - "ZarrV2ConsolidatedMetadataJSON": create_zarr_v2_consolidated_metadata_json, - "ZarrV2GroupMetadataJSON": create_zarr_v2_group_metadata_json, - "ZarrV2ZArrayJSON": create_zarr_v2_zarray_json, - "ZarrV2ZGroupJSON": create_zarr_v2_zgroup_json, - "ZarrV3ArrayMetadataJSON": create_zarr_v3_array_metadata_json, - "ZarrV3ConsolidatedMetadataJSON": create_zarr_v3_consolidated_metadata_json, - "ZarrV3GroupMetadataJSON": create_zarr_v3_group_metadata_json, -} - - -def test_every_document_typeddict_has_a_factory() -> None: - documents = { - name - for name in zarr_metadata.__all__ - if typing_extensions.is_typeddict(getattr(zarr_metadata, name)) - and name.endswith("JSON") - and name not in _HELPER_SHAPES - } - assert documents == set(_FACTORIES) - for factory in _FACTORIES.values(): - assert factory.__name__ in zarr_metadata.builder.__all__ - - -# -- error cases, one test per failure mode ---------------------------------- - - -def test_error_v3_array_semantic_rules_run() -> None: - with pytest.raises(MetadataValidationError, match=r"\[0, 255\]"): - create_zarr_v3_array_metadata_json(**{**V3_ARRAY, "fill_value": 300}) - - -def test_error_v3_array_structural_garbage_from_untyped_caller() -> None: - with pytest.raises(MetadataValidationError) as info: - create_zarr_v3_array_metadata_json(zarr_format="3") # type: ignore[arg-type] - assert {p.loc[0] for p in info.value.problems if p.kind == "missing_key"} >= { - "node_type", - "shape", - } - - -def test_error_v3_array_extension_shadows_standard_key() -> None: - with pytest.raises(MetadataValidationError, match="standard metadata key"): - create_zarr_v3_array_metadata_json(**V3_ARRAY, extensions={"shape": (9,)}) - - -def test_error_v3_group_extension_shadows_standard_key() -> None: - with pytest.raises(MetadataValidationError, match="standard metadata key"): - create_zarr_v3_group_metadata_json( - zarr_format=3, node_type="group", extensions={"attributes": {}} - ) - - -def test_error_v3_consolidated_invalid() -> None: - with pytest.raises(MetadataValidationError): - create_zarr_v3_consolidated_metadata_json( - kind="inline", - must_understand=True, - metadata={}, # type: ignore[typeddict-item] - ) - - -def test_error_v3_consolidated_child_violates_composition_rules() -> None: - child = {**V3_ARRAY, "fill_value": 300} - with pytest.raises(MetadataValidationError) as exc_info: - create_zarr_v3_consolidated_metadata_json( - kind="inline", must_understand=False, metadata={"a": child} - ) - assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ - (("metadata", "a", "fill_value"), "invalid_value") - ] - - -def test_error_v2_array_structural() -> None: - with pytest.raises(MetadataValidationError): - create_zarr_v2_array_metadata_json(**{**V2_ZARRAY, "order": "K"}) # type: ignore[typeddict-item] - - -def test_error_v3_array_malformed_raw_dtype() -> None: - # r names outside the family grammar are misspellings of a known - # family, not unknown extensions, and must not escape judgment. - with pytest.raises(MetadataValidationError, match="positive multiple of 8"): - create_zarr_v3_array_metadata_json(**{**V3_ARRAY, "data_type": "r12", "fill_value": (1,)}) - - -def test_error_v2_zarray_attributes_via_splat() -> None: - # The signature excludes `attributes` statically, but a splatted call - # bypasses that; the runtime backstop must hold the strict shape. - with pytest.raises(MetadataValidationError, match=".zattrs"): - create_zarr_v2_zarray_json(**{**V2_ZARRAY, "attributes": {"unit": "m"}}) - - -def test_error_v2_zgroup_attributes_via_splat() -> None: - splatted: dict[str, object] = {"zarr_format": 2, "attributes": {"unit": "m"}} - with pytest.raises(MetadataValidationError, match=".zattrs"): - create_zarr_v2_zgroup_json(**splatted) - - -def test_error_v2_consolidated_envelope() -> None: - with pytest.raises(MetadataValidationError, match="expected a mapping"): - create_zarr_v2_consolidated_metadata_json( - zarr_consolidated_format=1, - metadata="not a mapping", # type: ignore[typeddict-item] - ) - - -def test_error_v2_consolidated_format_is_not_one() -> None: - with pytest.raises(MetadataValidationError) as exc_info: - create_zarr_v2_consolidated_metadata_json( - zarr_consolidated_format=2, - metadata={}, - ) - assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ - (("zarr_consolidated_format",), "invalid_value") - ] - - -def test_error_v2_consolidated_array_entry_is_invalid() -> None: - with pytest.raises(MetadataValidationError) as exc_info: - create_zarr_v2_consolidated_metadata_json( - zarr_consolidated_format=1, - metadata={"foo/.zarray": {}}, # type: ignore[typeddict-item] - ) - assert any( - problem.loc[:2] == ("metadata", "foo/.zarray") and problem.kind == "missing_key" - for problem in exc_info.value.problems - ) - - -def test_error_v2_consolidated_entry_has_unknown_suffix() -> None: - with pytest.raises(MetadataValidationError, match="metadata file suffix"): - create_zarr_v2_consolidated_metadata_json( - zarr_consolidated_format=1, - metadata={"foo/data": {}}, # type: ignore[typeddict-item] - ) diff --git a/packages/zarr-metadata/tests/rules/test_rule_properties.py b/packages/zarr-metadata/tests/rules/test_rule_properties.py index 9f014c524a..e89dc80644 100644 --- a/packages/zarr-metadata/tests/rules/test_rule_properties.py +++ b/packages/zarr-metadata/tests/rules/test_rule_properties.py @@ -8,9 +8,9 @@ from hypothesis import given from hypothesis import strategies as st -from zarr_metadata.builder import create_zarr_v3_array_metadata_json from zarr_metadata.model import MetadataValidationError from zarr_metadata.rules import ( + parse_array_metadata_v3, validate_array_metadata_v2, validate_array_metadata_v3, validate_group_metadata_v2, @@ -148,7 +148,7 @@ def test_nested_sharding_pipelines_accept_divisible_inner_chunks(exponents: list data_type=st.sampled_from(("int8", "uint8", "int16", "uint16", "int32", "uint32")), fill_value=st.integers(min_value=-(2**40), max_value=2**40), ) -def test_validator_and_factory_agree(data_type: str, fill_value: int) -> None: +def test_validator_and_parser_agree(data_type: str, fill_value: int) -> None: document: Mapping[str, object] = { "zarr_format": 3, "node_type": "array", @@ -162,13 +162,13 @@ def test_validator_and_factory_agree(data_type: str, fill_value: int) -> None: validator_accepts = validate_array_metadata_v3(document) == () try: - create_zarr_v3_array_metadata_json(**document) # type: ignore[arg-type] + parse_array_metadata_v3(document) except MetadataValidationError: - factory_accepts = False + parser_accepts = False else: - factory_accepts = True + parser_accepts = True - assert validator_accepts == factory_accepts + assert validator_accepts == parser_accepts @pytest.mark.parametrize("position", ["data_type", "cast_value"]) From c70091959fd8a860945e54e3b072b90696e83f60 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:10:14 +0200 Subject: [PATCH 007/107] refactor(zarr-metadata)!: drop the check_* result family A fourth read-side front door for a question `validate_*` and `parse_*` already answer, with two new public types (`Valid`, `Invalid`) and no consumer. Removed with its tests and changelog fragment. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.5.md | 17 --- .../src/zarr_metadata/rules/__init__.py | 18 +-- .../src/zarr_metadata/rules/_result.py | 101 ----------------- .../zarr-metadata/tests/rules/test_result.py | 107 ------------------ .../zarr-metadata/tests/test_public_api.py | 3 - 5 files changed, 1 insertion(+), 245 deletions(-) delete mode 100644 packages/zarr-metadata/changes/4379.feature.5.md delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_result.py delete mode 100644 packages/zarr-metadata/tests/rules/test_result.py diff --git a/packages/zarr-metadata/changes/4379.feature.5.md b/packages/zarr-metadata/changes/4379.feature.5.md deleted file mode 100644 index 1a5433c683..0000000000 --- a/packages/zarr-metadata/changes/4379.feature.5.md +++ /dev/null @@ -1,17 +0,0 @@ -Added `check_*` entry points in `zarr_metadata.rules` returning a -discriminated `Valid[T] | Invalid`, for callers who want a document and -its problems in one value. - -The literal `valid` field narrows to either the normalized document or a -nonempty problem tuple: - -```python -result = check_array_metadata_v3(loaded) -if result.valid: - store(result.document) # typed ZarrV3ArrayMetadataJSON -else: - report(result.problems) # non-empty tuple of problems -``` - -Use `validate_*` to collect problems and `parse_*` to raise on invalid -input. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py index 0631f838c8..9916ec734f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py @@ -3,7 +3,7 @@ `zarr_metadata.model` checks JSON structure. This module also checks cross-field constraints such as fill-value compatibility, codec ordering, and dimension counts. Its `validate_*`, `is_*`, and `parse_*` -functions mirror the model API; `check_*` returns `Valid[T] | Invalid`. +functions mirror the model API. Rules target canonical metadata and may be stricter than readers that coerce inputs. Unknown entity names are left unjudged. Known entities @@ -27,15 +27,6 @@ validate_group_metadata_v3, ) from zarr_metadata.rules._engine import Rule, RuleCheck, applicable, run_rules -from zarr_metadata.rules._result import ( - Invalid, - Valid, - ValidationResult, - check_array_metadata_v2, - check_array_metadata_v3, - check_group_metadata_v2, - check_group_metadata_v3, -) from zarr_metadata.rules._v2_array import ZARR_V2_ARRAY, ZARR_V2_ARRAY_RULES from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY, ZARR_V3_ARRAY_RULES from zarr_metadata.rules._v3_group import ZARR_V3_GROUP, ZARR_V3_GROUP_RULES @@ -47,16 +38,9 @@ "ZARR_V3_ARRAY_RULES", "ZARR_V3_GROUP", "ZARR_V3_GROUP_RULES", - "Invalid", "Rule", "RuleCheck", - "Valid", - "ValidationResult", "applicable", - "check_array_metadata_v2", - "check_array_metadata_v3", - "check_group_metadata_v2", - "check_group_metadata_v3", "is_array_metadata_v2", "is_array_metadata_v3", "is_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_result.py b/packages/zarr-metadata/src/zarr_metadata/rules/_result.py deleted file mode 100644 index 4b099c6947..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_result.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Tagged validation results. - -`check_*` returns `Valid[T] | Invalid`. Testing the literal `valid` -field narrows to either the normalized document or a nonempty problem -tuple. Use `validate_*` to collect problems and `parse_*` to raise on -invalid input. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Generic, Literal, TypeAlias, TypeVar, cast - -from zarr_metadata.model._validation import ValidationProblem, arrays_to_tuples -from zarr_metadata.rules._documents import ( - validate_array_metadata_v2, - validate_array_metadata_v3, - validate_group_metadata_v2, - validate_group_metadata_v3, -) -from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON # noqa: TC001 -from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON # noqa: TC001 -from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON # noqa: TC001 -from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON # noqa: TC001 - -DocumentT = TypeVar("DocumentT") - - -@dataclass(frozen=True, slots=True) -class Valid(Generic[DocumentT]): - """A document that passed structural and composition validation.""" - - document: DocumentT - valid: Literal[True] = True - - -@dataclass(frozen=True, slots=True) -class Invalid: - """Every reason a document failed validation. - - `problems` is never empty: an empty report is a `Valid`. - """ - - problems: tuple[ValidationProblem, ...] - valid: Literal[False] = False - - def __post_init__(self) -> None: - if len(self.problems) == 0: - msg = "Invalid requires at least one validation problem" - raise ValueError(msg) - - -ValidationResult: TypeAlias = Valid[DocumentT] | Invalid -"""Either a validated document or the problems that disqualified it.""" - - -def check_array_metadata_v3(value: object) -> ValidationResult[ZarrV3ArrayMetadataJSON]: - """`value` as a valid v3 array document, or the problems disqualifying it. - - A `Valid` carries the normalized document (JSON arrays as tuples), - exactly as `parse_array_metadata_v3` returns it. - """ - problems = validate_array_metadata_v3(value) - if len(problems) != 0: - return Invalid(problems) - return Valid(cast("ZarrV3ArrayMetadataJSON", arrays_to_tuples(value))) - - -def check_array_metadata_v2(value: object) -> ValidationResult[ZarrV2ArrayMetadataJSON]: - """`value` as a valid v2 array document, or the problems disqualifying it.""" - problems = validate_array_metadata_v2(value) - if len(problems) != 0: - return Invalid(problems) - return Valid(cast("ZarrV2ArrayMetadataJSON", arrays_to_tuples(value))) - - -def check_group_metadata_v3(value: object) -> ValidationResult[ZarrV3GroupMetadataJSON]: - """`value` as a valid v3 group document, or the problems disqualifying it.""" - problems = validate_group_metadata_v3(value) - if len(problems) != 0: - return Invalid(problems) - return Valid(cast("ZarrV3GroupMetadataJSON", arrays_to_tuples(value))) - - -def check_group_metadata_v2(value: object) -> ValidationResult[ZarrV2GroupMetadataJSON]: - """`value` as a valid v2 group document, or the problems disqualifying it.""" - problems = validate_group_metadata_v2(value) - if len(problems) != 0: - return Invalid(problems) - return Valid(cast("ZarrV2GroupMetadataJSON", arrays_to_tuples(value))) - - -__all__ = [ - "Invalid", - "Valid", - "ValidationResult", - "check_array_metadata_v2", - "check_array_metadata_v3", - "check_group_metadata_v2", - "check_group_metadata_v3", -] diff --git a/packages/zarr-metadata/tests/rules/test_result.py b/packages/zarr-metadata/tests/rules/test_result.py deleted file mode 100644 index 49e859a873..0000000000 --- a/packages/zarr-metadata/tests/rules/test_result.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Tests for the `check_*` tagged-union entry points.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, get_args, get_type_hints - -import pytest - -from zarr_metadata.rules import ( - Invalid, - Valid, - ValidationResult, - check_array_metadata_v2, - check_array_metadata_v3, - check_group_metadata_v2, - check_group_metadata_v3, -) -from zarr_metadata.rules._documents import validate_array_metadata_v3 - -if TYPE_CHECKING: - from collections.abc import Callable, Mapping - -V3_ARRAY: Mapping[str, object] = { - "zarr_format": 3, - "node_type": "array", - "shape": (4, 4), - "data_type": "uint8", - "fill_value": 0, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, - "chunk_key_encoding": "default", - "codecs": ("bytes",), -} -V2_ARRAY: Mapping[str, object] = { - "zarr_format": 2, - "shape": (4,), - "chunks": (2,), - "dtype": " None: - result = check(doc) - assert isinstance(result, Valid) - assert result.valid is True - assert result.document == doc - - -def test_valid_normalizes_like_parse() -> None: - # A Valid carries the canonical document, not the caller's spelling. - result = check_array_metadata_v3({**V3_ARRAY, "shape": [4, 4], "codecs": ["bytes"]}) - assert isinstance(result, Valid) - assert result.document["shape"] == (4, 4) - assert result.document["codecs"] == ("bytes",) - - -def test_error_invalid_carries_every_problem() -> None: - result = check_array_metadata_v3({**V3_ARRAY, "node_type": "grid", "fill_value": 300}) - assert isinstance(result, Invalid) - assert result.valid is False - assert len(result.problems) != 0 - # the same report validate_* would give, not a summary of it - assert result.problems == validate_array_metadata_v3( - {**V3_ARRAY, "node_type": "grid", "fill_value": 300} - ) - - -def test_error_invalid_cannot_have_an_empty_report() -> None: - with pytest.raises(ValueError, match="at least one"): - Invalid(()) - - -def test_public_result_type_is_runtime_subscriptable() -> None: - assert get_args(ValidationResult[int]) == (Valid[int], Invalid) - - -def test_public_check_annotations_resolve_at_runtime() -> None: - hints = get_type_hints(check_array_metadata_v3) - assert get_args(hints["return"])[1] is Invalid - - -def test_discriminant_narrows_both_ways() -> None: - # The point of the union: `valid` selects which member is readable. - good = check_array_metadata_v3(V3_ARRAY) - if good.valid: - assert good.document["zarr_format"] == 3 - else: # pragma: no cover - the fixture is valid - pytest.fail("expected a Valid result") - - bad = check_array_metadata_v3({**V3_ARRAY, "fill_value": 300}) - if bad.valid: # pragma: no cover - the fixture is invalid - pytest.fail("expected an Invalid result") - else: - assert any(problem.loc == ("fill_value",) for problem in bad.problems) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 89be982958..ca880c0070 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -287,7 +287,6 @@ def test_all_is_grouped_and_unique() -> None: "HexFloat32", "HexFloat64", "JSONValue", - "Invalid", "MetadataValidationError", "NumpyDatetime64", "NumpyTimeUnit", @@ -301,9 +300,7 @@ def test_all_is_grouped_and_unique() -> None: "ShardingIndexLocation", "Struct", "StructField", - "Valid", "ValidationProblem", - "ValidationResult", } ) From 25504e06f9d5c9f23ffcea5c91517b00e3c782d3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:12:21 +0200 Subject: [PATCH 008/107] refactor(zarr-metadata)!: drop rules.is_*, normalize documents once The `is_*` counterparts were `bool`, not `TypeIs`, and each docstring had to explain that they do not do what the name promises; `model.is_*` remains for narrowing. Their removal leaves `validate_*` and `parse_*`, which now share `_judged` over an already-normalized document instead of walking it twice per call. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.md | 10 +- packages/zarr-metadata/docs/api/index.md | 4 +- .../src/zarr_metadata/rules/__init__.py | 12 +-- .../src/zarr_metadata/rules/_documents.py | 95 +++++++------------ .../tests/rules/test_documents.py | 39 ++++---- 5 files changed, 58 insertions(+), 102 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index f8326ac6b1..638d88bb81 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -18,19 +18,19 @@ module there and changes nothing else. (chunks/shape rank agreement) and `ZARR_V3_GROUP_RULES` (inline consolidated metadata recurses, judging each embedded child document by its own rules at its path). -- **Read-side trios**: `validate_*` / `is_*` / `parse_*` for array and +- **Read-side entry points**: `validate_*` / `parse_*` for array and group documents in both format versions mirror the model layer's grammar with a stronger judgment — structure *and* composition, every problem reported together, JSON arrays normalized to tuples before - judgment. The `is_*` functions deliberately return `bool` rather than - `TypeIs`: a composition-invalid document is still an instance of the - TypedDict, so only the structural layer can narrow honestly. + judgment. There is no `is_*` counterpart: a composition-invalid + document is still an instance of the TypedDict, so only the structural + layer can narrow honestly — use `zarr_metadata.model.is_*` for that. - **Boundary change**: two composition checks that lived in the structural validator moved here — v3 `dimension_names` vs `shape` and v2 `chunks` vs `shape` rank agreement. `zarr_metadata.model`'s validators, parsers, and dataclasses now accept those documents (they are lossless, structurally well-formed representations of what a store - may contain); use the `rules` trios to judge them. This also removes + may contain); use the `rules` validators to judge them. This also removes the double report the overlap used to produce. - **Strictness stance**, now documented on the package: `zarr_metadata` models canonical documents and is deliberately stricter than any given diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 788cffece8..7d42570f29 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -10,8 +10,8 @@ The package is organized to mirror the structure of the Zarr specifications: structural validators, loc-aware parsers, and the `UNSET` sentinel - [`zarr_metadata.rules`](rules.md) — composition rules: cross-field judgments over full documents (fill value vs. data type, codec pipeline - ordering, chunk geometry), plus whole-document `validate`/`is`/`parse` - trios combining structure and composition + ordering, chunk geometry), plus whole-document `validate`/`parse` + entry points combining structure and composition - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py index 9916ec734f..80fc4ee3d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py @@ -2,8 +2,8 @@ `zarr_metadata.model` checks JSON structure. This module also checks cross-field constraints such as fill-value compatibility, codec -ordering, and dimension counts. Its `validate_*`, `is_*`, and `parse_*` -functions mirror the model API. +ordering, and dimension counts. Its `validate_*` and `parse_*` functions +mirror the model API. Rules target canonical metadata and may be stricter than readers that coerce inputs. Unknown entity names are left unjudged. Known entities @@ -13,10 +13,6 @@ """ from zarr_metadata.rules._documents import ( - is_array_metadata_v2, - is_array_metadata_v3, - is_group_metadata_v2, - is_group_metadata_v3, parse_array_metadata_v2, parse_array_metadata_v3, parse_group_metadata_v2, @@ -41,10 +37,6 @@ "Rule", "RuleCheck", "applicable", - "is_array_metadata_v2", - "is_array_metadata_v3", - "is_group_metadata_v2", - "is_group_metadata_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", "parse_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index d88ff862d0..e997eb423a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -1,14 +1,15 @@ """Whole-document structural and composition validation. -These `validate_*`, `is_*`, and `parse_*` functions mirror the model API -but apply both validation layers. The `is_*` functions return `bool`, not -`TypeIs`: composition validity is stricter than TypedDict membership. -Use `zarr_metadata.model.is_*` for type narrowing. +These `validate_*` and `parse_*` functions mirror the model API but apply +both validation layers. There is deliberately no `is_*` counterpart: +composition validity is stricter than TypedDict membership, so a guard +here could not narrow honestly. Use `zarr_metadata.model.is_*` for type +narrowing. """ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, cast from zarr_metadata.model._validation import ( @@ -33,12 +34,31 @@ from zarr_metadata.rules._v3_group import ZARR_V3_GROUP_RULES if TYPE_CHECKING: + from collections.abc import Callable + from zarr_metadata.model._validation import ValidationProblem + from zarr_metadata.rules._engine import Rule from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON + _StructuralValidator = Callable[[object], tuple[ValidationProblem, ...]] + + +def _judged( + normalized: object, structure: _StructuralValidator, rules: Sequence[Rule] +) -> tuple[ValidationProblem, ...]: + """Structural and composition problems in an already-normalized document. + + Takes the normalized value rather than the caller's input so that + `validate_*` and `parse_*` each walk the document once. + """ + problems = structure(normalized) + if isinstance(normalized, Mapping): + problems = problems + run_rules(rules, cast("Mapping[str, object]", normalized)) + return tuple(problems) + def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a valid v3 array document. @@ -49,21 +69,7 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: (e.g. fresh `json.loads` output) are judged at the canonical data level rather than rejected for their spelling. """ - normalized = arrays_to_tuples(value) - problems = _validate_structure_v3(normalized) - if isinstance(normalized, Mapping): - document = cast("Mapping[str, object]", normalized) - problems = problems + run_rules(ZARR_V3_ARRAY_RULES, document) - return tuple(problems) - - -def is_array_metadata_v3(value: object) -> bool: - """Whether `value` is a structurally and compositionally valid v3 array doc. - - Deliberately not a `TypeIs` guard — see the module docstring. Use - `zarr_metadata.model.is_array_metadata_v3` to narrow. - """ - return len(validate_array_metadata_v3(value)) == 0 + return _judged(arrays_to_tuples(value), _validate_structure_v3, ZARR_V3_ARRAY_RULES) def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: @@ -74,7 +80,7 @@ def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: problem found. """ normalized = arrays_to_tuples(value) - problems = validate_array_metadata_v3(normalized) + problems = _judged(normalized, _validate_structure_v3, ZARR_V3_ARRAY_RULES) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3ArrayMetadataJSON", normalized) @@ -86,20 +92,7 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: JSON arrays are normalized to tuples before judgment, as in `validate_array_metadata_v3`. """ - normalized = arrays_to_tuples(value) - problems = _validate_structure_v2(normalized) - if isinstance(normalized, Mapping): - document = cast("Mapping[str, object]", normalized) - problems = problems + run_rules(ZARR_V2_ARRAY_RULES, document) - return tuple(problems) - - -def is_array_metadata_v2(value: object) -> bool: - """Whether `value` is a structurally and compositionally valid v2 array doc. - - Deliberately not a `TypeIs` guard — see the module docstring. - """ - return len(validate_array_metadata_v2(value)) == 0 + return _judged(arrays_to_tuples(value), _validate_structure_v2, ZARR_V2_ARRAY_RULES) def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: @@ -110,7 +103,7 @@ def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: problem found. """ normalized = arrays_to_tuples(value) - problems = validate_array_metadata_v2(normalized) + problems = _judged(normalized, _validate_structure_v2, ZARR_V2_ARRAY_RULES) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV2ArrayMetadataJSON", normalized) @@ -123,26 +116,13 @@ def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: consolidated child document invalid under its own rules is reported here, at its path. """ - normalized = arrays_to_tuples(value) - problems = _validate_group_structure_v3(normalized) - if isinstance(normalized, Mapping): - document = cast("Mapping[str, object]", normalized) - problems = problems + run_rules(ZARR_V3_GROUP_RULES, document) - return tuple(problems) - - -def is_group_metadata_v3(value: object) -> bool: - """Whether `value` is a structurally and compositionally valid v3 group doc. - - Deliberately not a `TypeIs` guard — see the module docstring. - """ - return len(validate_group_metadata_v3(value)) == 0 + return _judged(arrays_to_tuples(value), _validate_group_structure_v3, ZARR_V3_GROUP_RULES) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: """Return `value` as a valid `ZarrV3GroupMetadataJSON`, or raise.""" normalized = arrays_to_tuples(value) - problems = validate_group_metadata_v3(normalized) + problems = _judged(normalized, _validate_group_structure_v3, ZARR_V3_GROUP_RULES) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3GroupMetadataJSON", normalized) @@ -154,28 +134,19 @@ def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: v2 group documents carry no composition constraints today, so this is the structural judgment, offered here for a uniform read-side API. """ - return _validate_group_structure_v2(arrays_to_tuples(value)) - - -def is_group_metadata_v2(value: object) -> bool: - """Whether `value` is a valid v2 group document (merged form).""" - return len(validate_group_metadata_v2(value)) == 0 + return _judged(arrays_to_tuples(value), _validate_group_structure_v2, ()) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: """Return `value` as a valid `ZarrV2GroupMetadataJSON`, or raise.""" normalized = arrays_to_tuples(value) - problems = validate_group_metadata_v2(normalized) + problems = _judged(normalized, _validate_group_structure_v2, ()) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV2GroupMetadataJSON", normalized) __all__ = [ - "is_array_metadata_v2", - "is_array_metadata_v3", - "is_group_metadata_v2", - "is_group_metadata_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", "parse_group_metadata_v2", diff --git a/packages/zarr-metadata/tests/rules/test_documents.py b/packages/zarr-metadata/tests/rules/test_documents.py index 94a49b6171..0e34ef1a2b 100644 --- a/packages/zarr-metadata/tests/rules/test_documents.py +++ b/packages/zarr-metadata/tests/rules/test_documents.py @@ -1,4 +1,4 @@ -"""Tests for the whole-document validation trios in `zarr_metadata.rules`.""" +"""Tests for the whole-document validators in `zarr_metadata.rules`.""" from __future__ import annotations @@ -11,8 +11,6 @@ is_array_metadata_v3 as model_is_array_metadata_v3, ) from zarr_metadata.rules import ( - is_array_metadata_v2, - is_array_metadata_v3, parse_array_metadata_v2, parse_array_metadata_v3, validate_array_metadata_v2, @@ -25,11 +23,11 @@ from zarr_metadata import ZarrV2ArrayMetadataJSON, ZarrV3ArrayMetadataJSON from zarr_metadata.model import ValidationProblem - # The trios are uniform in their inputs (any object) and differ only in - # the document type they hand back, which these tests never depend on. + # The validators are uniform in their inputs (any object) and differ + # only in the document type they hand back, which these tests never + # depend on. Validator = Callable[[object], tuple[ValidationProblem, ...]] Parser = Callable[[object], Mapping[str, object]] - Check = Callable[[object], bool] V3_ARRAY: ZarrV3ArrayMetadataJSON = { "zarr_format": 3, @@ -53,38 +51,32 @@ "filters": None, } -# (validate, parse, is_, document) — every entry must validate cleanly -# through the combined trio; list-spelled arrays check that parse -# normalizes. Error paths get their own tests below. -CASES: dict[str, tuple[Validator, Parser, Check, Mapping[str, object]]] = { +# (validate, parse, document) — every entry must validate cleanly through +# both entry points; list-spelled arrays check that parse normalizes. +# Error paths get their own tests below. +CASES: dict[str, tuple[Validator, Parser, Mapping[str, object]]] = { "v3-array": ( validate_array_metadata_v3, parse_array_metadata_v3, - is_array_metadata_v3, V3_ARRAY, ), "v3-array-list-spelled": ( validate_array_metadata_v3, parse_array_metadata_v3, - is_array_metadata_v3, {**V3_ARRAY, "shape": [4, 4], "codecs": ["bytes"]}, ), "v2-array": ( validate_array_metadata_v2, parse_array_metadata_v2, - is_array_metadata_v2, V2_ARRAY, ), } -@pytest.mark.parametrize(("validate", "parse", "check", "doc"), CASES.values(), ids=list(CASES)) -def test_valid_documents( - validate: Validator, parse: Parser, check: Check, doc: Mapping[str, object] -) -> None: +@pytest.mark.parametrize(("validate", "parse", "doc"), CASES.values(), ids=list(CASES)) +def test_valid_documents(validate: Validator, parse: Parser, doc: Mapping[str, object]) -> None: parsed = parse(doc) assert validate(parsed) == () - assert check(parsed) is True shape = doc["shape"] assert isinstance(shape, (list, tuple)) assert parsed["shape"] == tuple(shape) @@ -118,10 +110,11 @@ def test_error_v2_parse_raises() -> None: parse_array_metadata_v2({**V2_ARRAY, "chunks": (2, 2)}) -def test_is_functions_are_not_type_guards() -> None: - # A composition-invalid document is still an instance of the TypedDict, - # so the model layer's TypeIs narrows it while the rules layer's plain - # bool judges it. Divergence here is the design, not a bug. +def test_composition_invalid_document_still_satisfies_the_typeddict() -> None: + # The model layer's TypeIs narrows a composition-invalid document, + # which is why the rules layer offers no guard of its own: a + # fill_value out of range does not stop the value being an instance + # of ZarrV3ArrayMetadataJSON. doc = {**V3_ARRAY, "fill_value": 300} assert model_is_array_metadata_v3(doc) is True - assert is_array_metadata_v3(doc) is False + assert validate_array_metadata_v3(doc) != () From f16bcea5ac3d31d59d840a14fd43d17144c1623d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:13:12 +0200 Subject: [PATCH 009/107] refactor(zarr-metadata): trim duplicated and unused rules-layer surface - `initial_spec` was a four-line tail of `chain_initial_spec` living in another module; folded into its only caller. - `STORAGE_TRANSFORMERS` was defined and exported without a reader. - `_engine`'s prior-art bibliography condensed to the claim it supports. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/rules/_engine.py | 33 ++++--------------- .../src/zarr_metadata/rules/_registry.py | 7 ++-- .../src/zarr_metadata/rules/_spec.py | 14 -------- .../src/zarr_metadata/v3/_extension_points.py | 2 -- 4 files changed, 12 insertions(+), 44 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py b/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py index bb101bd301..924a705fe2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py @@ -1,38 +1,19 @@ """Rules gated by the document fields they read. -A `Rule` runs when every key in `requires` is present. The same rule set -therefore supports complete documents and partial builders without -imposing field order. - -Prior art ---------- -The gate is conventional, which is the point. Ecto's -`Ecto.Changeset.validate_change/3` invokes a validator "only if a change -for the given field exists", so one changeset serves both full inserts -and partial updates. Clojure spec's two-phase `s/keys` separates -required-key presence from key/value conformance precisely because "we -routinely deal with optional and partial data". Valibot's `partialCheck` -takes the paths a cross-field rule reads and runs it "whenever the -selected part of the data is valid". Presence-conditional rules-as-data -are JSON Schema's `dependentSchemas` / `dependentRequired` applicators. - -- https://hexdocs.pm/ecto/Ecto.Changeset.html -- https://clojure.org/about/spec -- https://valibot.dev/api/partialCheck/ -- https://www.learnjsonschema.com/2020-12/applicator/dependentschemas/ - -Two consequences follow from gating rather than ordering. +A `Rule` runs when every key in `requires` is present, so one rule set +serves complete documents and partial ones without imposing field order. +Gating rather than ordering is conventional — Ecto changesets, Clojure +spec, Valibot's `partialCheck`, JSON Schema's `dependentSchemas` — and +has two consequences here. **Order-free by construction.** No topological sort, so mutually -dependent rules are expressible — unlike Yup, whose equivalent `deps` -orders rules and therefore rejects cycles outright. +dependent rules are expressible. **Absence is deliberately inexpressible.** A rule cannot ask whether a field is missing: that is negation-as-failure, sound only under a closed-world assumption, and a partially built document is an open world where the key may still arrive. Required-key checks therefore stay in -structural validation — the same stratification Ecto -(`validate_required`), spec (`:req`), and JSON Schema (`required`) apply. +structural validation. Rules may receive structurally invalid values. A rule that cannot safely interpret its inputs leaves the problem to structural validation. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py index 042d3542bb..8ebef4ef40 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Final, cast from zarr_metadata.rules._engine import Rule, as_string_mapping, prefixed -from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec, initial_spec, propagate +from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec, propagate from zarr_metadata.v3._extension_points import CHUNK_GRID, ExtensionPointField, canonical_name from zarr_metadata.v3._shape import ( blocking_problems, @@ -302,7 +302,10 @@ def chain_initial_spec(document: Mapping[str, object]) -> ArraySpec: values = cast("tuple[object, ...]", extents) if all(isinstance(v, int) and not isinstance(v, bool) and v >= 1 for v in values): chunk_shape = cast("tuple[int, ...]", values) - return initial_spec(document, chunk_shape) + data_type = document.get("data_type") + if not isinstance(data_type, (str, Mapping)): + data_type = None + return ArraySpec(chunk_shape, data_type) # type: ignore[arg-type] __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py index 88459c26b4..158b590089 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py @@ -132,24 +132,10 @@ def propagate( spec = spec.with_shape(None) -def initial_spec(document: Mapping[str, object], chunk_shape: tuple[int, ...] | None) -> ArraySpec: - """The spec entering a document's top-level codec chain. - - The array a chunk pipeline encodes is one chunk, so the incoming shape - is the chunk grid's chunk shape (`None` if the grid is not a regular - grid this package can read). The data type is the document's own. - """ - data_type = document.get("data_type") - if not isinstance(data_type, (str, Mapping)): - data_type = None - return ArraySpec(chunk_shape, data_type) # type: ignore[arg-type] - - __all__ = [ "NOTHING_KNOWN", "ArraySpec", "SpecTransition", - "initial_spec", "propagate", "spec_transition", "transitions_registered", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py index 03a8472a2b..631a9e705f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py @@ -25,7 +25,6 @@ CHUNK_GRID: Final[ExtensionPointField] = "chunk_grid" CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" CODECS: Final[ExtensionPointField] = "codecs" -STORAGE_TRANSFORMERS: Final[ExtensionPointField] = "storage_transformers" RAW_BYTES_FAMILY: Final = "r" """Canonical key for the parameterized raw-bytes data type family. @@ -48,7 +47,6 @@ def canonical_name(field: ExtensionPointField, name: str) -> str: "CODECS", "DATA_TYPE", "RAW_BYTES_FAMILY", - "STORAGE_TRANSFORMERS", "ExtensionPointField", "canonical_name", ] From f0bc51d493e8f22588b0456464c4a13588b86dbb Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:18:59 +0200 Subject: [PATCH 010/107] refactor(zarr-metadata): one storage classifier for bytes and struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The struct spec states its field constraints in the bytes codec's terms rather than inventing its own: field types must have "a fixed encoded size", and "When a `struct` type contains multi-byte numeric fields, the `bytes` codec MUST be configured with an explicit `endian` setting". So the struct rule's fixed-size question and the bytes rule's endianness question are one classification, and the two hand-written tables of data-type sizes were the same table twice. `rules._storage_class` now owns it, keyed off the data-type modules' own name constants, and `test_registry_drift` fails if a new data type arrives without a class — previously it would have silently gone unjudged by both rules. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../rules/_entities/bytes_codec.py | 79 +--------- .../rules/_entities/struct_dtype.py | 66 ++------ .../src/zarr_metadata/rules/_storage_class.py | 141 ++++++++++++++++++ .../tests/test_registry_drift.py | 22 +++ 4 files changed, 178 insertions(+), 130 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py index 048ce33fc7..611acb8e51 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py @@ -2,15 +2,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal, cast +from typing import TYPE_CHECKING from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._engine import as_string_mapping from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.rules._storage_class import data_type_name, storage_class from zarr_metadata.v3._extension_points import CODECS, DATA_TYPE from zarr_metadata.v3._shape import blocking_problems, validate_known_entity_metadata from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME -from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN if TYPE_CHECKING: from collections.abc import Mapping @@ -19,72 +18,6 @@ _ARRAY_V3 = "zarr_v3_array" -_SINGLE_BYTE = frozenset({"bool", "int8", "uint8"}) -_MULTI_BYTE = frozenset( - { - "int16", - "int32", - "int64", - "uint16", - "uint32", - "uint64", - "float16", - "float32", - "float64", - "complex64", - "complex128", - "numpy.datetime64", - "numpy.timedelta64", - } -) -_VARIABLE_LENGTH = frozenset({"bytes", "string"}) -_StorageClass = Literal["single_byte", "multi_byte", "variable_length"] - - -def _data_type_name(data_type: object) -> str | None: - if isinstance(data_type, str): - return data_type - mapping = as_string_mapping(data_type) - if mapping is None: - return None - name = mapping.get("name") - return name if isinstance(name, str) else None - - -def _storage_class(data_type: object) -> _StorageClass | None: - """Classify known data types by their raw byte representation.""" - name = _data_type_name(data_type) - if name in _SINGLE_BYTE or (name is not None and RAW_BYTES_NAME_PATTERN.fullmatch(name)): - return "single_byte" - if name in _MULTI_BYTE: - return "multi_byte" - if name in _VARIABLE_LENGTH: - return "variable_length" - if name != "struct": - return None - - envelope = as_string_mapping(data_type) - configuration = ( - as_string_mapping(envelope.get("configuration")) if envelope is not None else None - ) - fields = configuration.get("fields") if configuration is not None else None - if not isinstance(fields, tuple): - return None - classes: list[_StorageClass] = [] - for field in cast("tuple[object, ...]", fields): - field_mapping = as_string_mapping(field) - if field_mapping is None or "data_type" not in field_mapping: - return None - field_class = _storage_class(field_mapping["data_type"]) - if field_class is None: - return None - classes.append(field_class) - if "variable_length" in classes: - return "variable_length" - if "multi_byte" in classes: - return "multi_byte" - return "single_byte" - @entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME) def data_type_has_a_raw_byte_representation( @@ -95,9 +28,9 @@ def data_type_has_a_raw_byte_representation( shape_verdict = validate_known_entity_metadata(DATA_TYPE, incoming.data_type) if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0: return () - storage_class = _storage_class(incoming.data_type) - if storage_class == "variable_length": - name = _data_type_name(incoming.data_type) + found = storage_class(incoming.data_type) + if found == "variable_length": + name = data_type_name(incoming.data_type) return ( ValidationProblem( (), @@ -105,7 +38,7 @@ def data_type_has_a_raw_byte_representation( "invalid_value", ), ) - if storage_class == "multi_byte" and "endian" not in configuration: + if found == "multi_byte" and "endian" not in configuration: return ( ValidationProblem( ("endian",), diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py index 8be61b0361..b6c4e7e823 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py @@ -3,6 +3,13 @@ `StructField`'s own docstring promises field names are unique within a struct and non-empty. Neither is expressible in a TypedDict, so both are composition judgments and live here. + +The fixed-size field rule is the `bytes` codec's question asked from the +other side — the spec writes it as "Variable-length data types (e.g. +`"string"`) MUST NOT be used as field types, as they do not have a fixed +encoded size" — so it defers to the shared classifier in +`zarr_metadata.rules._storage_class` rather than keeping a second table +of data-type sizes. """ from __future__ import annotations @@ -13,8 +20,8 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.rules._engine import as_string_mapping from zarr_metadata.rules._registry import entity_rule, run_entity_rules +from zarr_metadata.rules._storage_class import storage_class from zarr_metadata.v3._extension_points import DATA_TYPE -from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME if TYPE_CHECKING: @@ -22,28 +29,6 @@ _ARRAY_V3 = "zarr_v3_array" -_FIXED_SIZE_NAMES = frozenset( - { - "bool", - "int8", - "int16", - "int32", - "int64", - "uint8", - "uint16", - "uint32", - "uint64", - "float16", - "float32", - "float64", - "complex64", - "complex128", - "numpy.datetime64", - "numpy.timedelta64", - } -) -_VARIABLE_SIZE_NAMES = frozenset({"bytes", "string"}) - @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) def field_data_types_obey_their_rules( @@ -84,39 +69,6 @@ def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str], return tuple(named) -def _known_fixed_size(data_type: object) -> bool | None: - """Whether a known data type is fixed-size; None means unknown.""" - if isinstance(data_type, str): - name = data_type - envelope = None - else: - envelope = as_string_mapping(data_type) - raw_name = envelope.get("name") if envelope is not None else None - name = raw_name if isinstance(raw_name, str) else None - if name in _FIXED_SIZE_NAMES or ( - isinstance(name, str) and RAW_BYTES_NAME_PATTERN.fullmatch(name) - ): - return True - if name in _VARIABLE_SIZE_NAMES: - return False - if name != STRUCT_DATA_TYPE_NAME or envelope is None: - return None - nested_configuration = as_string_mapping(envelope.get("configuration")) - fields = nested_configuration.get("fields") if nested_configuration is not None else None - if not isinstance(fields, tuple): - return None - results: list[bool] = [] - for field in cast("tuple[object, ...]", fields): - field_mapping = as_string_mapping(field) - if field_mapping is None or "data_type" not in field_mapping: - return None - result = _known_fixed_size(field_mapping["data_type"]) - if result is None: - return None - results.append(result) - return all(results) - - @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) def fields_are_non_empty( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec @@ -137,7 +89,7 @@ def field_data_types_are_fixed_size( field_mapping = as_string_mapping(field) if field_mapping is None or "data_type" not in field_mapping: continue - if _known_fixed_size(field_mapping["data_type"]) is False: + if storage_class(field_mapping["data_type"]) == "variable_length": problems.append( ValidationProblem( ("fields", index, "data_type"), diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py b/packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py new file mode 100644 index 0000000000..19f9e2119b --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py @@ -0,0 +1,141 @@ +"""How a data type lays out in bytes, and what that demands of a codec. + +Two rules in this package need the same fact about a data type: whether +one of its scalars occupies a fixed number of bytes, and if so whether +that number is one (no byte order to declare) or more (a byte order the +`bytes` codec must declare). + +The `bytes` codec spec makes `endian` "Required for data types for which +endianness is applicable... multi-byte data types, such as `uint16` and +`int32`, but not single-byte data types, such as `uint8` or `bool`", and +addresses fixed-size numeric types only. + +The `struct` spec then states its own constraints in those same terms +rather than inventing new ones: a field's data type must be one "whose +size in bytes is fixed and known at the time the array is opened", since +variable-length types "do not have a fixed encoded size"; and "When a +`struct` type contains multi-byte numeric fields, the `bytes` codec MUST +be configured with an explicit `endian` setting", while a struct +"composed entirely of single-byte fields... MAY omit the `endian` +configuration". + +So a struct's own storage class is the widest class among its fields, +recursively, and "valid as a struct field" is simply "not +variable-length". One classifier answers both. + +- https://zarr-specs.readthedocs.io/en/latest/v3/codecs/bytes/index.html +- https://github.com/zarr-developers/zarr-extensions/blob/main/data-types/struct/README.md +""" + +from __future__ import annotations + +from typing import Literal, cast + +from zarr_metadata.rules._engine import as_string_mapping +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME + +StorageClass = Literal["single_byte", "multi_byte", "variable_length"] +"""How one scalar of a data type occupies bytes. + +`single_byte` and `multi_byte` are both fixed-size; they differ only in +whether a byte order applies. +""" + +_SINGLE_BYTE = frozenset({BOOL_DATA_TYPE_NAME, INT8_DATA_TYPE_NAME, UINT8_DATA_TYPE_NAME}) +_MULTI_BYTE = frozenset( + { + INT16_DATA_TYPE_NAME, + INT32_DATA_TYPE_NAME, + INT64_DATA_TYPE_NAME, + UINT16_DATA_TYPE_NAME, + UINT32_DATA_TYPE_NAME, + UINT64_DATA_TYPE_NAME, + FLOAT16_DATA_TYPE_NAME, + FLOAT32_DATA_TYPE_NAME, + FLOAT64_DATA_TYPE_NAME, + COMPLEX64_DATA_TYPE_NAME, + COMPLEX128_DATA_TYPE_NAME, + NUMPY_DATETIME64_DATA_TYPE_NAME, + NUMPY_TIMEDELTA64_DATA_TYPE_NAME, + } +) +_VARIABLE_LENGTH = frozenset({BYTES_DATA_TYPE_NAME, STRING_DATA_TYPE_NAME}) + + +def data_type_name(data_type: object) -> str | None: + """The name a data-type metadata field carries, or None if it has none.""" + if isinstance(data_type, str): + return data_type + mapping = as_string_mapping(data_type) + if mapping is None: + return None + name = mapping.get("name") + return name if isinstance(name, str) else None + + +def storage_class(data_type: object) -> StorageClass | None: + """Classify a known data type by its raw byte representation. + + None means undetermined — an unknown name, or a `struct` whose fields + this package cannot read — and every rule declines rather than + guessing. A `struct` takes the widest class among its fields, so a + struct of `uint8` and `int32` is `multi_byte` and one containing a + `string` is `variable_length`, recursively. + """ + name = data_type_name(data_type) + if name in _SINGLE_BYTE or (name is not None and RAW_BYTES_NAME_PATTERN.fullmatch(name)): + return "single_byte" + if name in _MULTI_BYTE: + return "multi_byte" + if name in _VARIABLE_LENGTH: + return "variable_length" + if name != STRUCT_DATA_TYPE_NAME: + return None + + envelope = as_string_mapping(data_type) + configuration = ( + as_string_mapping(envelope.get("configuration")) if envelope is not None else None + ) + fields = configuration.get("fields") if configuration is not None else None + if not isinstance(fields, tuple): + return None + classes: set[StorageClass] = set() + for field in cast("tuple[object, ...]", fields): + field_mapping = as_string_mapping(field) + if field_mapping is None or "data_type" not in field_mapping: + return None + field_class = storage_class(field_mapping["data_type"]) + if field_class is None: + return None + classes.add(field_class) + if "variable_length" in classes: + return "variable_length" + if "multi_byte" in classes: + return "multi_byte" + return "single_byte" + + +__all__ = [ + "StorageClass", + "data_type_name", + "storage_class", +] diff --git a/packages/zarr-metadata/tests/test_registry_drift.py b/packages/zarr-metadata/tests/test_registry_drift.py index f21851b990..04418b5fef 100644 --- a/packages/zarr-metadata/tests/test_registry_drift.py +++ b/packages/zarr-metadata/tests/test_registry_drift.py @@ -14,6 +14,9 @@ import zarr_metadata.v3.chunk_key_encoding import zarr_metadata.v3.codec import zarr_metadata.v3.data_type +from zarr_metadata.rules._storage_class import ( # pyright: ignore[reportPrivateUsage] + storage_class, +) from zarr_metadata.rules._v3_array import ( _check_fill_for_dtype, # pyright: ignore[reportPrivateUsage] ) @@ -83,3 +86,22 @@ def test_every_data_type_has_a_fill_value_branch() -> None: name for name in {*dtype_names, "r8"} if _check_fill_for_dtype(name, object()) is None } assert not unjudged + + +def test_every_data_type_module_has_a_storage_class() -> None: + # The bytes codec's endianness rule and the struct field rule both ask + # this question, so an unclassified data type silently disables both. + # `struct` classifies from its fields, so it is sampled with one; the + # r family has no name constant and is represented by "r8". + dtype_names = _module_constants(zarr_metadata.v3.data_type, "_DATA_TYPE_NAME") + assert dtype_names, "constant scan found nothing — the naming convention moved?" + samples: dict[str, object] = { + "struct": { + "name": "struct", + "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}, + } + } + unclassified = { + name for name in {*dtype_names, "r8"} if storage_class(samples.get(name, name)) is None + } + assert not unclassified From a930714a1db51cad93c40df61bcb098656216a26 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:27:42 +0200 Subject: [PATCH 011/107] docs(zarr-metadata): commit to strict unknown keys, and test it The strict reading was already the behavior; the prose hedged, telling callers they "can filter" a kind that every raising path rejects. State the decision instead: an unmodelled member of a known entity's configuration is almost always a typo or a setting meant for a different entity, so `parse_*` and the pydantic field types both refuse it, and the dedicated kind is there for triage via `validate_*`. Two tests pin what was untested: that the pydantic field types run the rules layer at all, and that they reject an unknown configuration member. Also syncs the README and docs feature lists, which still described the pydantic integration as delegating to the model parser and omitted the rules layer entirely, and notes that the generated JSON Schemas leave configurations open, so schema-valid input can still fail at runtime. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/README.md | 24 +++++++++++---- .../zarr-metadata/changes/4379.feature.6.md | 11 ++++--- packages/zarr-metadata/docs/index.md | 26 +++++++++++++---- .../src/zarr_metadata/model/_validation.py | 16 +++++----- .../tests/model/test_pydantic_module.py | 29 +++++++++++++++++++ 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index da0052a0c6..cf10d99651 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -6,7 +6,7 @@ Documentation: ## What this is -Two layers and an optional integration: +Three layers and an optional integration: - **Typed JSON shapes**: `TypedDict` definitions and `Literal` aliases for the JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html) @@ -17,9 +17,13 @@ Two layers and an optional integration: models of whole metadata documents, with structural validators, loc-aware parsers, and store-key (de)serialization. A document produced by `to_json` shares no mutable state with the model that produced it. +- **Composition rules** (`zarr_metadata.rules`): cross-field judgments over + whole documents — fill value against data type, codec pipeline ordering, + chunk geometry — with `validate_*` / `parse_*` entry points that apply + structure and composition together. - **Optional Pydantic integration** (`zarr_metadata.pydantic`, requires - Pydantic 2.13 or newer): each model as a Pydantic field type that validates - raw documents through the same strict parser. + Pydantic 2.13 or newer): each model as a Pydantic field type that runs raw + documents through the rules layer. ## What this is for @@ -72,14 +76,24 @@ ambiguous input as it sees fit and then validate the canonical result. Nothing here decides whether a data type, chunk grid, codec, or storage transformer is *supported*; that belongs to consumer implementations. +An unmodelled member inside a *known* entity's `configuration` is an error +under that strict reading, in `parse_*` and in the Pydantic field types +alike: such a member is almost always a typo or a setting meant for a +different entity, and accepting it silently means silently ignoring what +the writer asked for. It carries its own `unknown_key` problem kind, so a +consumer who wants the tolerant reading can collect problems with +`validate_*` and filter that kind out. + The Pydantic integration's generated JSON Schemas express independently checkable document structure and field constraints, but they are not a replacement for runtime model validation. Standard JSON Schema treats a mathematically integral number such as `1.0` as an integer, while the runtime boundary requires Python `int` values, and it cannot express arbitrary same-length relations such as `dimension_names` versus `shape` or v2 `chunks` -versus `shape`. Consumers should run the runtime validators after schema -validation. +versus `shape`. The generated schemas also leave every `configuration` +open, so a schema-valid document can still be rejected at runtime for an +unmodelled configuration member. Consumers should run the runtime +validators after schema validation. ## Scope diff --git a/packages/zarr-metadata/changes/4379.feature.6.md b/packages/zarr-metadata/changes/4379.feature.6.md index fe9d67eced..e163ed5445 100644 --- a/packages/zarr-metadata/changes/4379.feature.6.md +++ b/packages/zarr-metadata/changes/4379.feature.6.md @@ -4,10 +4,13 @@ problem kind, and no longer suppress the other rules about that entity. Whether configurations are closed remains unspecified ([zarr-specs#270](https://github.com/zarr-developers/zarr-specs/issues/270)), -so this package retains its strict reading with two safeguards: - -- callers can filter the dedicated `unknown_key` kind; -- unknown keys do not suppress other rules for the same entity. +and this package retains its strict reading deliberately: in practice an +unmodelled member is a typo or a setting meant for a different entity, and +accepting it silently means silently ignoring what the writer asked for. +Every entry point that raises rejects it, the pydantic field types +included. The dedicated kind exists so that a consumer who wants the +tolerant reading can collect problems with `validate_*` and filter, and so +that an unknown key never masks the other findings about its entity. Model round-trips preserve unmodeled members. Shape-exact `TypeIs` guards still reject them because the corresponding TypedDicts are closed. diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index 2368094796..42df058d7b 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -32,28 +32,34 @@ closely model the content of the Zarr specifications, such as: validators, loc-aware parsers, and store-key (de)serialization. A document produced by `to_json` shares no mutable state with the model that produced it. +- **Composition rules** ([`zarr_metadata.rules`](api/rules.md)): cross-field + judgments over whole documents — fill value against data type, codec + pipeline ordering, chunk geometry — with `validate_*` / `parse_*` entry + points that apply structure and composition together. - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), requires Pydantic 2.13 or newer): each model as a Pydantic field type that - validates raw documents through the same strict parser. + runs raw documents through the rules layer. ## What this is for The public `TypedDict` definitions describe the static JSON shape of Zarr -metadata. For strict, loc-aware validation of JSON loaded from disk, use the -model parser: +metadata. To judge JSON loaded from disk, structure and composition together, +use the rules layer; to get a normalized document model, use the model parser: ```python import json from zarr_metadata.model import ZarrV3ArrayMetadata +from zarr_metadata.rules import parse_array_metadata_v3 with open("zarr.json", "rb") as f: raw = json.load(f) -metadata = ZarrV3ArrayMetadata.from_json(raw) +document = parse_array_metadata_v3(raw) # raises with every problem found +metadata = ZarrV3ArrayMetadata.from_json(document) ``` -The optional Pydantic integration delegates raw input to the same strict -parser and returns the same normalized model class: +The optional Pydantic integration runs raw input through the rules layer +and returns the same normalized model class: ```python from pydantic import TypeAdapter @@ -85,6 +91,14 @@ ambiguous input as it sees fit and then validate the canonical result. Nothing here decides whether a data type, chunk grid, codec, or storage transformer is *supported*; that belongs to consumer implementations. +An unmodelled member inside a *known* entity's `configuration` is an error +under that strict reading, in `parse_*` and in the Pydantic field types +alike: such a member is almost always a typo or a setting meant for a +different entity, and accepting it silently means silently ignoring what +the writer asked for. It carries its own `unknown_key` problem kind, so a +consumer who wants the tolerant reading can collect problems with +`validate_*` and filter that kind out. + ## Scope At minimum, this library supports what Zarr-Python needs: the complete diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 95c0822c59..34351a6f15 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -39,13 +39,15 @@ - `invalid_json`: bytes that do not decode as JSON. - `unknown_key`: a member this package does not model appears inside an entity whose shape it does model (e.g. an extra key in a `blosc` - configuration). Distinguished from `invalid_value` because the Zarr v3 - spec does not say whether a `configuration` is closed - (zarr-developers/zarr-specs#270 has been open since 2023), so this is - the package's strict reading rather than a definite violation: a - document carrying one is very likely fine, just written by something - that models more than we do. Callers that prefer tolerance can filter - this kind out; the package itself never lets it mask other findings. + configuration). Whether a `configuration` is closed is unspecified + (zarr-developers/zarr-specs#270 has been open since 2023), and this + package takes the strict reading: in practice such a member is a typo, + or a setting meant for a different entity, and accepting it silently + means silently ignoring what the writer asked for. Every entry point + that raises rejects it, the pydantic field types included. It gets a + kind of its own so that a caller who wants the tolerant reading can + collect problems with `rules.validate_*` and filter, and so that it + never masks the other findings about the same entity. """ diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 80067e81ae..302a7447ae 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -265,6 +265,35 @@ def test_metadata_field_serializes_shorthand_and_false_object() -> None: ) == {"name": "optional", "must_understand": False} +def test_field_types_apply_composition_rules() -> None: + """The field types judge composition, not just structure. + + `fill_value: 300` is a well-formed JSON integer, so only the rules + layer can reject it for a `uint8` array. + """ + document = {**V3_ARRAY_DOC, "data_type": "uint8", "fill_value": 300} + with pytest.raises(ValidationError, match="fill_value"): + TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(document) + + +def test_field_types_reject_unknown_configuration_members() -> None: + """An unmodelled member of a known entity's configuration is an error. + + Whether a `configuration` is closed is unspecified + (zarr-developers/zarr-specs#270), and this integration takes the + strict reading deliberately: in practice such a member is a typo or a + setting meant for a different codec, and silently accepting it means + silently ignoring what the writer asked for. Callers who want the + tolerant reading use `rules.validate_*` and filter `unknown_key`. + """ + document = { + **V3_ARRAY_DOC, + "codecs": ({"name": "bytes", "configuration": {"endian": "little", "endain": "big"}},), + } + with pytest.raises(ValidationError, match="unexpected key 'endain'"): + TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(document) + + def test_core_package_does_not_import_pydantic() -> None: """Importing zarr_metadata (in a fresh interpreter) must not import pydantic: the integration is opt-in via zarr_metadata.pydantic.""" From 744b8f1a2db66dcf1c774b18c13264e4d43f827e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 11:35:04 +0200 Subject: [PATCH 012/107] chore(zarr-metadata): review nits on the trimmed rules branch - Drop a `pyright: ignore[reportPrivateUsage]` on a public name; pyright is configured for `src` only, so the comment was never evaluated. - Keep `Sequence` under TYPE_CHECKING alongside `Callable` and `Rule`. - Rewrap the changelog fragment and say "two validation layers", so it does not read as contradicting the README's three-layer overview. - "different entity", consistently, in the pydantic strictness test. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.feature.md | 8 ++++---- .../zarr-metadata/src/zarr_metadata/rules/_documents.py | 4 ++-- .../zarr-metadata/tests/model/test_pydantic_module.py | 2 +- packages/zarr-metadata/tests/test_registry_drift.py | 4 +--- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index 638d88bb81..e1ce8faf95 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -1,10 +1,10 @@ Added `zarr_metadata.rules`: composition rules for full metadata -documents. The package now models metadata in two layers with one +documents. The package now validates metadata in two layers with one contract each — `model` checks structure element by element and `rules` judges composition across the document. Rules are registered where they -are defined; rules about a particular codec, chunk grid, or data type live with that entity under -`rules._entities` and are dispatched by name, so adding an entity adds a -module there and changes nothing else. +are defined; rules about a particular codec, chunk grid, or data type +live with that entity under `rules._entities` and are dispatched by name, +so adding an entity adds a module there and changes nothing else. - **Rule sets**: `ZARR_V3_ARRAY_RULES` covers fill value vs. data type, codec pipeline kind ordering, known-name shapes, diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index e997eb423a..8268ce9698 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -9,7 +9,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import TYPE_CHECKING, cast from zarr_metadata.model._validation import ( @@ -34,7 +34,7 @@ from zarr_metadata.rules._v3_group import ZARR_V3_GROUP_RULES if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.rules._engine import Rule diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 302a7447ae..22f90c4b48 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -282,7 +282,7 @@ def test_field_types_reject_unknown_configuration_members() -> None: Whether a `configuration` is closed is unspecified (zarr-developers/zarr-specs#270), and this integration takes the strict reading deliberately: in practice such a member is a typo or a - setting meant for a different codec, and silently accepting it means + setting meant for a different entity, and silently accepting it means silently ignoring what the writer asked for. Callers who want the tolerant reading use `rules.validate_*` and filter `unknown_key`. """ diff --git a/packages/zarr-metadata/tests/test_registry_drift.py b/packages/zarr-metadata/tests/test_registry_drift.py index 04418b5fef..b678c17e62 100644 --- a/packages/zarr-metadata/tests/test_registry_drift.py +++ b/packages/zarr-metadata/tests/test_registry_drift.py @@ -14,9 +14,7 @@ import zarr_metadata.v3.chunk_key_encoding import zarr_metadata.v3.codec import zarr_metadata.v3.data_type -from zarr_metadata.rules._storage_class import ( # pyright: ignore[reportPrivateUsage] - storage_class, -) +from zarr_metadata.rules._storage_class import storage_class from zarr_metadata.rules._v3_array import ( _check_fill_for_dtype, # pyright: ignore[reportPrivateUsage] ) From 43d73f729fdd430a52af9b6c0522b76364a85bbf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 12:10:48 +0200 Subject: [PATCH 013/107] docs(zarr-metadata)!: correct the scope of the unknown-key stance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wording — "every entry point that raises rejects it, the pydantic field types included" — is false. Three raising entry points accept an unmodelled configuration member: `model.parse_*`, `ZarrV3ArrayMetadata.from_json`, and the bare `ZarrV3MetadataField` pydantic type. Only the rules layer judges configurations, so only `rules.parse_*` and the whole-document pydantic field types reject it. Say that instead, in the README, the docs site, the `ProblemKind` docstring and the changelog fragment, and assert the `ZarrV3MetadataField` half of the boundary so it cannot move unnoticed. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/README.md | 18 ++++++++++++------ .../zarr-metadata/changes/4379.feature.6.md | 11 +++++++---- packages/zarr-metadata/docs/index.md | 18 ++++++++++++------ .../src/zarr_metadata/model/_validation.py | 12 +++++++----- .../tests/model/test_pydantic_module.py | 11 +++++++++++ 5 files changed, 49 insertions(+), 21 deletions(-) diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index cf10d99651..91530e644a 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -77,12 +77,18 @@ Nothing here decides whether a data type, chunk grid, codec, or storage transformer is *supported*; that belongs to consumer implementations. An unmodelled member inside a *known* entity's `configuration` is an error -under that strict reading, in `parse_*` and in the Pydantic field types -alike: such a member is almost always a typo or a setting meant for a -different entity, and accepting it silently means silently ignoring what -the writer asked for. It carries its own `unknown_key` problem kind, so a -consumer who wants the tolerant reading can collect problems with -`validate_*` and filter that kind out. +under that strict reading: such a member is almost always a typo or a +setting meant for a different entity, and accepting it silently means +silently ignoring what the writer asked for. It carries its own +`unknown_key` problem kind, so a consumer who wants the tolerant reading +can collect problems with `validate_*` and filter that kind out. + +Judging it is the rules layer's job, so it is `zarr_metadata.rules` and +the whole-document Pydantic field types (which run the rules layer) that +reject it. The model layer never interpreted entity configurations and +still does not, so `model.parse_*` and `from_json` accept such a +document; so does the bare `ZarrV3MetadataField` Pydantic type, which +judges one metadata field and carries no composition rules. The Pydantic integration's generated JSON Schemas express independently checkable document structure and field constraints, but they are not a diff --git a/packages/zarr-metadata/changes/4379.feature.6.md b/packages/zarr-metadata/changes/4379.feature.6.md index e163ed5445..e39cb20165 100644 --- a/packages/zarr-metadata/changes/4379.feature.6.md +++ b/packages/zarr-metadata/changes/4379.feature.6.md @@ -7,10 +7,13 @@ Whether configurations are closed remains unspecified and this package retains its strict reading deliberately: in practice an unmodelled member is a typo or a setting meant for a different entity, and accepting it silently means silently ignoring what the writer asked for. -Every entry point that raises rejects it, the pydantic field types -included. The dedicated kind exists so that a consumer who wants the -tolerant reading can collect problems with `validate_*` and filter, and so -that an unknown key never masks the other findings about its entity. +Judging it is the rules layer's job: `rules.parse_*` and the +whole-document pydantic field types reject it, while `model.parse_*` and +the bare `ZarrV3MetadataField` type — neither of which interprets entity +configurations — accept it. The dedicated kind exists so that a consumer +who wants the tolerant reading can collect problems with `validate_*` and +filter, and so that an unknown key never masks the other findings about +its entity. Model round-trips preserve unmodeled members. Shape-exact `TypeIs` guards still reject them because the corresponding TypedDicts are closed. diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index 42df058d7b..38a66dafd3 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -92,12 +92,18 @@ Nothing here decides whether a data type, chunk grid, codec, or storage transformer is *supported*; that belongs to consumer implementations. An unmodelled member inside a *known* entity's `configuration` is an error -under that strict reading, in `parse_*` and in the Pydantic field types -alike: such a member is almost always a typo or a setting meant for a -different entity, and accepting it silently means silently ignoring what -the writer asked for. It carries its own `unknown_key` problem kind, so a -consumer who wants the tolerant reading can collect problems with -`validate_*` and filter that kind out. +under that strict reading: such a member is almost always a typo or a +setting meant for a different entity, and accepting it silently means +silently ignoring what the writer asked for. It carries its own +`unknown_key` problem kind, so a consumer who wants the tolerant reading +can collect problems with `validate_*` and filter that kind out. + +Judging it is the rules layer's job, so it is `zarr_metadata.rules` and +the whole-document Pydantic field types (which run the rules layer) that +reject it. The model layer never interpreted entity configurations and +still does not, so `model.parse_*` and `from_json` accept such a +document; so does the bare `ZarrV3MetadataField` Pydantic type, which +judges one metadata field and carries no composition rules. ## Scope diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 34351a6f15..3ba683136e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -43,11 +43,13 @@ (zarr-developers/zarr-specs#270 has been open since 2023), and this package takes the strict reading: in practice such a member is a typo, or a setting meant for a different entity, and accepting it silently - means silently ignoring what the writer asked for. Every entry point - that raises rejects it, the pydantic field types included. It gets a - kind of its own so that a caller who wants the tolerant reading can - collect problems with `rules.validate_*` and filter, and so that it - never masks the other findings about the same entity. + means silently ignoring what the writer asked for. Judging it belongs + to the rules layer, so `rules.parse_*` and the whole-document pydantic + field types reject it while `model.parse_*` — which never interpreted + configurations — accepts it. It gets a kind of its own so that a caller + who wants the tolerant reading can collect problems with + `rules.validate_*` and filter, and so that it never masks the other + findings about the same entity. """ diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 22f90c4b48..410692a054 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -285,6 +285,11 @@ def test_field_types_reject_unknown_configuration_members() -> None: setting meant for a different entity, and silently accepting it means silently ignoring what the writer asked for. Callers who want the tolerant reading use `rules.validate_*` and filter `unknown_key`. + + Scope: the whole-document field types run the rules layer and so + reject it. `ZarrV3MetadataField` judges one metadata field, carries no + composition rules, and accepts it — asserted below so the boundary + cannot move silently. """ document = { **V3_ARRAY_DOC, @@ -293,6 +298,12 @@ def test_field_types_reject_unknown_configuration_members() -> None: with pytest.raises(ValidationError, match="unexpected key 'endain'"): TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(document) + codec = {"name": "bytes", "configuration": {"endian": "little", "endain": "big"}} + assert TypeAdapter(zmp.ZarrV3MetadataField).validate_python(codec).configuration == { + "endian": "little", + "endain": "big", + } + def test_core_package_does_not_import_pydantic() -> None: """Importing zarr_metadata (in a fresh interpreter) must not import From b963667d97f87fcefb642a3ef0c94272970f7708 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 12:27:01 +0200 Subject: [PATCH 014/107] fix(zarr-metadata): close five gaps in the composition rules Found by adversarial review; each has a regression test. - scale_offset registers the identity spec transition the spec describes ("MUST be performed using the arithmetic semantics of the input array's data type"; astype was removed in v3 in favour of cast_value). Without it a no-op codec stopped propagation and stood down every later rule. The test exemption that recorded this is gone. - chain_initial_spec keeps the rank when it cannot keep the extents: an ArraySpec extent may now be None individually, so a rectilinear grid or a zero extent no longer hides a rank-mismatched transpose or shard. - Entity rules declare the configuration members they read, and run_entity_rules stands down only the rules that read an unusable member rather than the whole entity. This also makes the invariant the configuration["member"] accesses rely on explicit and checked at registration, where it was previously true only by inspection. - A rule reporting at the entity itself keeps that location instead of being re-based under a "configuration" node a bare-string entity does not have. - The endianness problem names the data type, so the shard-index case reads as uint64 rather than appearing to contradict the document. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.bugfix.md | 26 +++++ .../rules/_entities/bytes_codec.py | 8 +- .../rules/_entities/cast_value.py | 2 +- .../src/zarr_metadata/rules/_entities/gzip.py | 2 +- .../rules/_entities/numpy_time.py | 9 +- .../rules/_entities/rectilinear_grid.py | 10 +- .../rules/_entities/regular_grid.py | 10 +- .../rules/_entities/scale_offset.py | 44 ++++++++ .../zarr_metadata/rules/_entities/sharding.py | 20 ++-- .../rules/_entities/struct_dtype.py | 10 +- .../rules/_entities/transpose.py | 4 +- .../src/zarr_metadata/rules/_registry.py | 103 ++++++++++++++---- .../src/zarr_metadata/rules/_spec.py | 11 +- .../src/zarr_metadata/v3/_shape.py | 11 ++ .../tests/rules/test_spec_propagation.py | 7 +- .../tests/rules/test_v3_array_rules.py | 96 +++++++++++++++- 16 files changed, 318 insertions(+), 55 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.bugfix.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md new file mode 100644 index 0000000000..298044fe53 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -0,0 +1,26 @@ +Five gaps in the composition rules, found by adversarial review of the +rules layer: + +- A modelled `array -> array` codec with no registered spec transition + stopped spec propagation, standing down every rule after it. The + `scale_offset` codec had none, so inserting a no-op `scale_offset` + switched off the `bytes` codec's endianness requirement and every + shard's geometry check. It now registers the identity transition the + spec describes, and no codec is exempt from having one. +- A chunk grid whose extents this package cannot read (a rectilinear + grid, or a regular grid with a non-positive extent) discarded the + *rank* along with the extents, so a rank-mismatched `transpose` order + or inner chunk shape went unreported. Every chunk of an array has the + array's rank whatever the grid, so the rank is kept and only the + geometry checks stand down. +- One unusable configuration member suppressed every other rule about + the same entity — a misspelled `index_location` hid the problems in a + shard's inner pipelines. Rules now declare the members they read and + stand down individually, so the rest of the entity is still judged. +- Problems about an entity as a whole pointed at a `configuration` node + that a bare-string entity does not have (`("codecs", 0, + "configuration")` for `"codecs": ["bytes"]`). They now point at the + entity. +- The endianness message named no data type, so inside a shard's + `index_codecs` it appeared to be about the array's own type rather + than the shard index's `uint64`. It names the type. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py index 611acb8e51..8f4dc91c63 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py @@ -19,7 +19,7 @@ _ARRAY_V3 = "zarr_v3_array" -@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME, reads=frozenset({"endian"})) def data_type_has_a_raw_byte_representation( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -29,8 +29,8 @@ def data_type_has_a_raw_byte_representation( if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0: return () found = storage_class(incoming.data_type) + name = data_type_name(incoming.data_type) if found == "variable_length": - name = data_type_name(incoming.data_type) return ( ValidationProblem( (), @@ -39,10 +39,12 @@ def data_type_has_a_raw_byte_representation( ), ) if found == "multi_byte" and "endian" not in configuration: + # Name the type: inside a shard's `index_codecs` the array is the + # shard index, whose uint64 type appears nowhere in the document. return ( ValidationProblem( ("endian",), - "endian is required for a data type containing multi-byte values", + f"endian is required for data type {name!r}, which contains multi-byte values", "missing_key", ), ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py index a14b6c4639..24c1982cbe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py @@ -28,7 +28,7 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -@entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME) +@entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME, reads=frozenset({"data_type"})) def target_data_type_obeys_its_rules( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py index 542574d1ad..d73f3fb9fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py @@ -17,7 +17,7 @@ _ARRAY_V3 = "zarr_v3_array" -@entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME, reads=frozenset({"level"})) def level_is_in_range( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py index c5baed7d4e..934b488050 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py @@ -17,6 +17,7 @@ _ARRAY_V3 = "zarr_v3_array" _MAX_SCALE_FACTOR = 2**31 - 1 +_SCALE_FACTOR = frozenset({"scale_factor"}) def _scale_factor_is_in_range( @@ -34,5 +35,9 @@ def _scale_factor_is_in_range( ) -entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE_NAME)(_scale_factor_is_in_range) -entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE_NAME)(_scale_factor_is_in_range) +entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE_NAME, reads=_SCALE_FACTOR)( + _scale_factor_is_in_range +) +entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE_NAME, reads=_SCALE_FACTOR)( + _scale_factor_is_in_range +) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py index 15f170ee97..af41ab4e44 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py @@ -44,7 +44,7 @@ def _expanded_extent(spec: Sequence[object]) -> int | None: return total -@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME) +@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shapes"})) def chunk_extents_are_positive( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -85,7 +85,13 @@ def chunk_extents_are_positive( return tuple(problems) -@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, requires=frozenset({"shape"})) +@entity_rule( + _ARRAY_V3, + CHUNK_GRID, + RECTILINEAR_CHUNK_GRID_NAME, + requires=frozenset({"shape"}), + reads=frozenset({"chunk_shapes"}), +) def tiles_the_array( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py index 0a19b10133..51c8d8db56 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py @@ -17,7 +17,7 @@ _ARRAY_V3 = "zarr_v3_array" -@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME) +@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shape"})) def chunk_extents_are_positive( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -39,7 +39,13 @@ def chunk_extents_are_positive( ) -@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, requires=frozenset({"shape"})) +@entity_rule( + _ARRAY_V3, + CHUNK_GRID, + REGULAR_CHUNK_GRID_NAME, + requires=frozenset({"shape"}), + reads=frozenset({"chunk_shape"}), +) def chunks_every_dimension( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py new file mode 100644 index 0000000000..631b21225c --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py @@ -0,0 +1,44 @@ +"""Spec transition for the `scale_offset` codec. + +`scale_offset` subtracts an offset and multiplies by a scale, element by +element, and the spec requires the result to be representable in the +array's own data type: "The encoding and decoding transformations MUST be +performed using the arithmetic semantics of the input array's data type. +If any intermediate or final value produced during encoding or decoding +is not representable in that data type, implementations MUST treat this +as an error." + +So it changes neither the shape nor the data type, and its transition is +the identity. The spec is explicit that narrowing is somebody else's job: +the codec's `astype` field "was removed from the `scale_offset` codec in +favor of expressing data type conversion via a dedicated codec", because +"in Zarr V3, a `dtype` field is not needed — the data type of the input +to an array-array codec is determined by its location in the `codecs` +metadata". + +Registering this matters beyond tidiness. A modelled `array -> array` +codec with no transition is treated as unknown, which stops propagation +and silently stands down every rule downstream of it — so without this, +inserting a no-op `scale_offset` would switch off the `bytes` codec's +endianness requirement and every shard's geometry check. + +https://github.com/zarr-developers/zarr-extensions/blob/main/codecs/scale_offset/README.md +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr_metadata.rules._spec import spec_transition +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArraySpec + + +@spec_transition(SCALE_OFFSET_CODEC_NAME) +def preserves_the_array(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: + """Element-wise arithmetic in the input type: same shape, same type.""" + return incoming diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py index ccc74c1c01..97dce0c5a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py @@ -37,12 +37,15 @@ from collections.abc import Mapping _ARRAY_V3 = "zarr_v3_array" +_CHUNK_SHAPE = frozenset({"chunk_shape"}) +_PIPELINES = frozenset({"chunk_shape", "codecs", "index_codecs"}) +_INDEX_CODECS = frozenset({"index_codecs"}) _VARIABLE_SIZE_CODECS = frozenset( {BLOSC_CODEC_NAME, GZIP_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME, ZSTD_CODEC_NAME} ) -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE) def inner_chunk_extents_are_positive( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -58,15 +61,16 @@ def inner_chunk_extents_are_positive( ) -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE) def inner_chunks_tile_the_incoming_array( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: """The inner chunk must rank-match and evenly divide the array it receives. - Declines when the incoming shape is unknown — an unclassified codec - upstream, or a non-regular grid at the top level — rather than - guessing from the document. + Declines when the incoming array is unknown entirely — an unclassified + codec upstream — rather than guessing. A known rank with unknown + extents (a chunk grid this package cannot read) still supports the + rank check; only the divisibility check needs the extents. """ if incoming.shape is None: return () @@ -89,11 +93,11 @@ def inner_chunks_tile_the_incoming_array( "invalid_value", ) for position, (outer_extent, inner_extent) in enumerate(zip(outer, inner, strict=True)) - if inner_extent >= 1 and outer_extent % inner_extent != 0 + if outer_extent is not None and inner_extent >= 1 and outer_extent % inner_extent != 0 ) -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_PIPELINES) def inner_pipelines_are_pipelines( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -133,7 +137,7 @@ def inner_pipelines_are_pipelines( return tuple(problems) -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_INDEX_CODECS) def index_codecs_have_fixed_encoded_size( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py index b6c4e7e823..2de41bc63f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py @@ -30,7 +30,7 @@ _ARRAY_V3 = "zarr_v3_array" -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_data_types_obey_their_rules( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -69,7 +69,7 @@ def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str], return tuple(named) -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def fields_are_non_empty( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -79,7 +79,7 @@ def fields_are_non_empty( return (ValidationProblem(("fields",), "expected at least one struct field", "invalid_value"),) -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_data_types_are_fixed_size( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -100,7 +100,7 @@ def field_data_types_are_fixed_size( return tuple(problems) -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_names_are_non_empty( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -114,7 +114,7 @@ def field_names_are_non_empty( ) -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME) +@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_names_are_unique( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py index 61d98f0b4b..7e97a9d31a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py @@ -31,7 +31,7 @@ def permute_shape(configuration: Mapping[str, object], incoming: ArraySpec) -> A return incoming.with_shape(tuple(shape[axis] for axis in order)) -@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) def order_is_a_permutation( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: @@ -52,7 +52,7 @@ def order_is_a_permutation( ) -@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME) +@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) def order_matches_incoming_rank( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py index 8ebef4ef40..0d1ca203d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -12,21 +12,20 @@ from collections import defaultdict from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Final, cast +from typing import Final, cast -from zarr_metadata.rules._engine import Rule, as_string_mapping, prefixed +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._engine import Rule, as_string_mapping from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec, propagate from zarr_metadata.v3._extension_points import CHUNK_GRID, ExtensionPointField, canonical_name from zarr_metadata.v3._shape import ( blocking_problems, + entity_configuration_keys, entity_name, modelled_entities, validate_known_entity_metadata, ) -if TYPE_CHECKING: - from zarr_metadata.model._validation import ValidationProblem - EntityCheck = Callable[ [Mapping[str, object], Mapping[str, object], "ArraySpec"], "tuple[ValidationProblem, ...]", @@ -56,11 +55,19 @@ class EntityRule: `requires` are *document* keys the check reads beyond the entity itself (e.g. `shape`), gating the rule exactly as `Rule.requires` does. + + `reads` are the *configuration* members the check reads. A rule runs + only when none of them has a shape problem of its own, which is what + makes `configuration["level"]`-style access inside a check safe: a + member that is missing, or present with the wrong type, is reported + at `("configuration", "")` by the shape validator, and the + rules that read it stand down while the rest still run. """ field: str entity: str requires: frozenset[str] + reads: frozenset[str] check: EntityCheck @@ -126,6 +133,7 @@ def entity_rule( field: ExtensionPointField, entity: str, requires: frozenset[str] = frozenset(), + reads: frozenset[str] = frozenset(), ) -> Callable[[EntityCheck], EntityRule]: """Register a rule about one named entity within `document_type`. @@ -145,7 +153,15 @@ def decorate(check: EntityCheck) -> EntityRule: f"validator in zarr_metadata.v3._shape; such a rule could never fire" ) raise ValueError(msg) - rule = EntityRule(field=field, entity=entity, requires=requires, check=check) + modelled = entity_configuration_keys(field, entity) + unmodelled = reads - (modelled or frozenset()) + if len(unmodelled) != 0: + msg = ( + f"entity rule {check.__name__!r} declares reads={sorted(unmodelled)}, which " + f"{entity!r} does not model; such a member can never carry a value to read" + ) + raise ValueError(msg) + rule = EntityRule(field=field, entity=entity, requires=requires, reads=reads, check=check) _ENTITY_RULES[field, canonical_entity].append(rule) return rule @@ -194,31 +210,54 @@ def run_entity_rules( rules = _ENTITY_RULES.get((field, canonical_name(field, name))) if rules is None or len(rules) == 0: return () - # Entity rules read configuration members by name, so they may only run - # once the shape validator vouches those members exist and are typed. - configuration = entity_configuration(field, value) + verdict = validate_known_entity_metadata(field, value) + if verdict is None: + return () + blocking = blocking_problems(verdict) + # A problem at the entity or at `configuration` itself means there is no + # configuration to read; one at ("configuration", member) means that one + # member is unusable and the rules that read it must stand down, while + # every other rule about this entity still runs. + if any(len(problem.loc) < 2 for problem in blocking): + return () + unusable = frozenset( + str(problem.loc[1]) for problem in blocking if problem.loc[0] == "configuration" + ) + configuration = _configuration_of(value) if configuration is None: return () problems: list[ValidationProblem] = [] for rule in rules: if not rule.requires <= document.keys(): continue - problems.extend( - prefixed((*loc, "configuration"), rule.check(configuration, document, incoming)) - ) + if len(rule.reads & unusable) != 0: + continue + for found in rule.check(configuration, document, incoming): + # A rule that reports at the entity itself (an empty loc) is + # judging the whole entity, not a member of its configuration — + # and a bare-string entity has no `configuration` node to point at. + base = (*loc, "configuration") if len(found.loc) != 0 else loc + problems.append(ValidationProblem((*base, *found.loc), found.message, found.kind)) return tuple(problems) def entity_configuration(field: ExtensionPointField, value: object) -> Mapping[str, object] | None: - """`value`'s configuration if its modelled fields are usable, else None. + """`value`'s configuration if every modelled field is usable, else None. - Shared by the dispatchers and by rules that reach across entities - (sharding's nested pipelines). `unknown_key` problems do not make an - entity unusable; anything else does. + The all-or-nothing gate `propagate` needs: a spec transition reads the + configuration to compute what the next codec receives, so one unusable + member makes the whole outgoing spec a guess. `run_entity_rules` uses + the finer per-member gate instead. `unknown_key` problems do not make + an entity unusable; anything else does. """ verdict = validate_known_entity_metadata(field, value) if verdict is None or len(blocking_problems(verdict)) != 0: return None + return _configuration_of(value) + + +def _configuration_of(value: object) -> Mapping[str, object] | None: + """`value`'s configuration mapping, with no judgment of its contents.""" mapping = as_string_mapping(value) if mapping is None: # Bare-string metadata is the canonical spelling for entities whose @@ -282,26 +321,44 @@ def run_chain_rules( return tuple(problems) +def _rank_only(shape: object) -> tuple[int | None, ...] | None: + """A shape of the document's rank with every extent undetermined.""" + if not isinstance(shape, tuple): + return None + dimensions = cast("tuple[object, ...]", shape) + if not all(isinstance(v, int) and not isinstance(v, bool) for v in dimensions): + return None + return (None,) * len(dimensions) + + def chain_initial_spec(document: Mapping[str, object]) -> ArraySpec: """The spec entering a document's top-level codec chain. - The array a chunk pipeline encodes is one chunk: shape from a regular - grid this package can read (None otherwise), data type from the - document. Non-positive chunk extents yield None for the shape — the - grid's own values rule owns that complaint, and geometry against a - zero extent is noise on top of it. + The array a chunk pipeline encodes is one chunk: extents from a + regular grid this package can read, data type from the document. + + When the extents are unavailable — a grid this package does not model, + a rectilinear grid whose chunks differ, or a regular grid with a + non-positive extent — the rank survives them. Every chunk of an array + has the array's rank, so `shape` becomes a tuple of `None` of that + length rather than `None`, and rank rules keep working while the + geometry rules stand down. (Geometry against a zero extent would be + noise on top of the grid's own complaint; a rank mismatch is a + separate fault and is still worth reporting.) """ from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME grid = document.get("chunk_grid") - chunk_shape: tuple[int, ...] | None = None + chunk_shape: tuple[int | None, ...] | None = None if entity_name(grid) == REGULAR_CHUNK_GRID_NAME: configuration = entity_configuration(CHUNK_GRID, grid) extents = configuration.get("chunk_shape") if configuration is not None else None if isinstance(extents, tuple): values = cast("tuple[object, ...]", extents) if all(isinstance(v, int) and not isinstance(v, bool) and v >= 1 for v in values): - chunk_shape = cast("tuple[int, ...]", values) + chunk_shape = cast("tuple[int | None, ...]", values) + if chunk_shape is None: + chunk_shape = _rank_only(document.get("shape")) data_type = document.get("data_type") if not isinstance(data_type, (str, Mapping)): data_type = None diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py index 158b590089..9fd60ef0c9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py @@ -36,14 +36,21 @@ class ArraySpec: """The array a codec receives; a field is `None` when undetermined. + `shape` is `None` only when there is no array left to describe (past + the array->bytes boundary) or when nothing about it can be determined. + An individual *extent* may be `None` while the rank is known: every + chunk of an array has the array's rank, whatever the chunk grid, so a + grid this package cannot read still pins `len(shape)`. Rules that need + a rank may use one; rules that need an extent test it for `None`. + `data_type` is the metadata-field value verbatim (a bare name or a name/configuration object) because rules compare it by name. """ - shape: tuple[int, ...] | None + shape: tuple[int | None, ...] | None data_type: ZarrV3MetadataFieldJSON | None - def with_shape(self, shape: tuple[int, ...] | None) -> ArraySpec: + def with_shape(self, shape: tuple[int | None, ...] | None) -> ArraySpec: return replace(self, shape=shape) def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArraySpec: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py index a31cc04d52..68e81ae94e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py @@ -637,6 +637,16 @@ def validate_known_entity_metadata( return _validate_known_entity(value, name, shape, field.replace("_", " ").rstrip("s")) +def entity_configuration_keys(field: ExtensionPointField, name: str) -> frozenset[str] | None: + """Every configuration member `name` models at `field`, or None if unmodelled. + + The registry checks a rule's declared `reads` against this, so a rule + cannot claim to read a member that does not exist. + """ + shape = _ENTITY_SHAPES.get(field, {}).get(canonical_name(field, name)) + return None if shape is None else shape.config_keys + + def modelled_entities() -> frozenset[tuple[ExtensionPointField, str]]: """Every `(extension point, name)` with a shape validator.""" return frozenset((field, name) for field, shapes in _ENTITY_SHAPES.items() for name in shapes) @@ -658,6 +668,7 @@ def blocking_problems( __all__ = [ "blocking_problems", + "entity_configuration_keys", "entity_name", "modelled_entities", "validate_known_chunk_grid_metadata", diff --git a/packages/zarr-metadata/tests/rules/test_spec_propagation.py b/packages/zarr-metadata/tests/rules/test_spec_propagation.py index e61733e8fa..90ef9a2a13 100644 --- a/packages/zarr-metadata/tests/rules/test_spec_propagation.py +++ b/packages/zarr-metadata/tests/rules/test_spec_propagation.py @@ -145,9 +145,10 @@ def test_unknown_codec_yields_nothing_known() -> None: def test_every_array_array_codec_registers_a_transition() -> None: # A modelled array->array codec with no transition is treated as - # unknown and stops propagation — safe, but silently weaker than - # intended. Make it a decision, not an omission. - assert set(ARRAY_ARRAY_CODEC_NAMES) <= transitions_registered() | {"scale_offset"} + # unknown and stops propagation, standing down every rule downstream + # of it. No exemptions: a codec that changes nothing registers the + # identity and says so. + assert set(ARRAY_ARRAY_CODEC_NAMES) <= transitions_registered() def test_error_transition_for_a_non_array_array_codec() -> None: diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index e3403726d5..11d17d61aa 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -284,10 +284,13 @@ def test_error_bytes_requires_endian_for_multibyte_data() -> None: def test_error_bytes_rejects_variable_length_data_type() -> None: + # The problem is about the codec, not about a member of its + # configuration — and this codec is spelled as a bare string, so it has + # no `configuration` node for a loc to point into. loc, message = _sole_problem( {**BASE, "data_type": "string", "fill_value": "", "codecs": ("bytes",)} ) - assert loc == ("codecs", 0, "configuration") + assert loc == ("codecs", 0) assert "not compatible" in message @@ -551,3 +554,94 @@ def test_unknown_member_survives_a_round_trip() -> None: emitted = model.to_json() codec = emitted["codecs"][0] assert codec["configuration"]["numThreads"] == 4 + + +# -- gaps found by adversarial review ---------------------------------------- + + +def test_error_scale_offset_does_not_stand_down_later_rules() -> None: + # A no-op array->array codec used to stop spec propagation, silently + # switching off every rule after it. + loc, message = _sole_problem( + { + **BASE, + "data_type": "uint16", + "codecs": ( + {"name": "scale_offset", "configuration": {"offset": 0, "scale": 1}}, + "bytes", + ), + } + ) + assert loc == ("codecs", 1, "configuration", "endian") + assert "uint16" in message + + +def test_error_rank_is_judged_under_a_chunk_grid_with_unknown_extents() -> None: + # A rectilinear grid gives no single chunk shape, but every chunk still + # has the array's rank, so a rank-3 transpose over a 1-D array is a fault. + loc, message = _sole_problem( + { + **BASE, + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((2, 2), (2, 2))}, + }, + "codecs": ({"name": "transpose", "configuration": {"order": (0, 1, 2)}}, "bytes"), + } + ) + assert loc == ("codecs", 0, "configuration", "order") + assert "2 dimensions" in message + + +def test_error_an_unusable_member_does_not_mask_the_rest_of_the_entity() -> None: + # A bad index_location says nothing about whether the inner pipelines + # are readable, so the pipeline problem must still be reported. + problems = validate_array_metadata_v3( + { + **BASE, + "codecs": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (2, 2), + "codecs": ("bytes", "bytes"), + "index_codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + "index_location": "middle", + }, + }, + ), + } + ) + assert {problem.loc for problem in problems} == { + ("codecs", 0, "configuration", "index_location"), + ("codecs", 0, "configuration", "codecs", 1), + } + + +def test_error_a_bare_entity_reports_at_the_entity_not_a_missing_node() -> None: + # "bytes" has no `configuration` node, so a problem about the codec as a + # whole must not point into one. + loc, _ = _sole_problem({**BASE, "data_type": "string", "fill_value": "", "codecs": ("bytes",)}) + assert loc == ("codecs", 0) + + +def test_error_endian_message_names_the_shard_index_type() -> None: + # Inside index_codecs the array is the shard index, whose uint64 type + # appears nowhere in the document; the message has to say so. + loc, message = _sole_problem( + { + **BASE, + "codecs": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (2, 2), + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + "index_codecs": ("bytes",), + }, + }, + ), + } + ) + assert loc == ("codecs", 0, "configuration", "index_codecs", 0, "configuration", "endian") + assert "uint64" in message From ec413c370cd285603e3f469dafbb15b56e29b88c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 12:43:21 +0200 Subject: [PATCH 015/107] fix(zarr-metadata): sound reads contract, and stop must_understand masking Two defects in the per-member gate added in 08d335d71, both found by re-running the adversarial review against it. `run_entity_rules` decided "the configuration is unreadable" from the *length* of a problem's location, so `("must_understand",)` qualified and a malformed envelope flag stood down every composition rule for that entity. Test the two locations that actually mean it instead. `reads` promised that declaring a member makes `configuration[member]` safe, but it was validated against every modelled member rather than the required ones. A rule declaring an optional member still raised KeyError out of validate_*, which must never raise. `reads` now accepts only required members; `reads_optional` covers the presence-tested case (the bytes codec's `endian`), and registration refuses the unsafe spelling with a message naming the alternative. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.bugfix.md | 4 ++ .../rules/_entities/bytes_codec.py | 2 +- .../src/zarr_metadata/rules/_registry.py | 63 ++++++++++++++----- .../src/zarr_metadata/v3/_shape.py | 16 +++++ .../tests/rules/test_registry.py | 35 +++++++++++ .../tests/rules/test_v3_array_rules.py | 19 ++++++ 6 files changed, 121 insertions(+), 18 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md index 298044fe53..a73e21c9b4 100644 --- a/packages/zarr-metadata/changes/4379.bugfix.md +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -24,3 +24,7 @@ rules layer: - The endianness message named no data type, so inside a shard's `index_codecs` it appeared to be about the array's own type rather than the shard index's `uint64`. It names the type. +- A malformed `must_understand` — part of an entity's envelope, not of its + configuration — counted as "the configuration is unreadable" and stood + down every composition rule for that entity. Only a problem at the + entity itself or at `configuration` as a whole does that now. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py index 8f4dc91c63..3e5624beca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py @@ -19,7 +19,7 @@ _ARRAY_V3 = "zarr_v3_array" -@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME, reads=frozenset({"endian"})) +@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME, reads_optional=frozenset({"endian"})) def data_type_has_a_raw_byte_representation( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec ) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py index 0d1ca203d4..66c9b64075 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -22,6 +22,7 @@ blocking_problems, entity_configuration_keys, entity_name, + entity_required_configuration_keys, modelled_entities, validate_known_entity_metadata, ) @@ -56,18 +57,26 @@ class EntityRule: itself (e.g. `shape`), gating the rule exactly as `Rule.requires` does. - `reads` are the *configuration* members the check reads. A rule runs - only when none of them has a shape problem of its own, which is what - makes `configuration["level"]`-style access inside a check safe: a - member that is missing, or present with the wrong type, is reported - at `("configuration", "")` by the shape validator, and the - rules that read it stand down while the rest still run. + `reads` are the *required* configuration members the check subscripts. + A rule runs only when none of them has a shape problem of its own, + which is what makes `configuration["level"]` safe: a required member + that is absent or ill-typed is reported at `("configuration", member)` + and stands the rule down, while the rest of the entity is still + judged. Only required members may be declared here — an optional one + can be absent with nothing reported, so subscripting it would raise + out of a validator. + + `reads_optional` are modelled members the check tests for presence + rather than subscripting (`"endian" not in configuration`). They gate + the rule the same way; they are separate so that the subscript + guarantee above stays true by construction. """ field: str entity: str requires: frozenset[str] reads: frozenset[str] + reads_optional: frozenset[str] check: EntityCheck @@ -134,6 +143,7 @@ def entity_rule( entity: str, requires: frozenset[str] = frozenset(), reads: frozenset[str] = frozenset(), + reads_optional: frozenset[str] = frozenset(), ) -> Callable[[EntityCheck], EntityRule]: """Register a rule about one named entity within `document_type`. @@ -153,15 +163,32 @@ def decorate(check: EntityCheck) -> EntityRule: f"validator in zarr_metadata.v3._shape; such a rule could never fire" ) raise ValueError(msg) - modelled = entity_configuration_keys(field, entity) - unmodelled = reads - (modelled or frozenset()) + modelled = entity_configuration_keys(field, entity) or frozenset() + required = entity_required_configuration_keys(field, entity) or frozenset() + unmodelled = (reads | reads_optional) - modelled if len(unmodelled) != 0: msg = ( - f"entity rule {check.__name__!r} declares reads={sorted(unmodelled)}, which " + f"entity rule {check.__name__!r} declares {sorted(unmodelled)}, which " f"{entity!r} does not model; such a member can never carry a value to read" ) raise ValueError(msg) - rule = EntityRule(field=field, entity=entity, requires=requires, reads=reads, check=check) + optional = reads - required + if len(optional) != 0: + msg = ( + f"entity rule {check.__name__!r} declares reads={sorted(optional)}, which " + f"{entity!r} does not require; an absent optional member is reported by " + f"nothing, so subscripting it would raise out of a validator. Declare it as " + f"reads_optional and test for presence instead." + ) + raise ValueError(msg) + rule = EntityRule( + field=field, + entity=entity, + requires=requires, + reads=reads, + reads_optional=reads_optional, + check=check, + ) _ENTITY_RULES[field, canonical_entity].append(rule) return rule @@ -214,14 +241,16 @@ def run_entity_rules( if verdict is None: return () blocking = blocking_problems(verdict) - # A problem at the entity or at `configuration` itself means there is no - # configuration to read; one at ("configuration", member) means that one - # member is unusable and the rules that read it must stand down, while - # every other rule about this entity still runs. - if any(len(problem.loc) < 2 for problem in blocking): + # Only two locations mean there is no configuration to read: the entity + # itself, and `configuration` as a whole. Anything else is about one + # member — including `must_understand`, which is part of the envelope + # and says nothing about whether the configuration is readable. + if any(problem.loc in ((), ("configuration",)) for problem in blocking): return () unusable = frozenset( - str(problem.loc[1]) for problem in blocking if problem.loc[0] == "configuration" + str(problem.loc[1]) + for problem in blocking + if len(problem.loc) >= 2 and problem.loc[0] == "configuration" ) configuration = _configuration_of(value) if configuration is None: @@ -230,7 +259,7 @@ def run_entity_rules( for rule in rules: if not rule.requires <= document.keys(): continue - if len(rule.reads & unusable) != 0: + if len((rule.reads | rule.reads_optional) & unusable) != 0: continue for found in rule.check(configuration, document, incoming): # A rule that reports at the entity itself (an empty loc) is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py index 68e81ae94e..454bd44b76 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py @@ -647,6 +647,21 @@ def entity_configuration_keys(field: ExtensionPointField, name: str) -> frozense return None if shape is None else shape.config_keys +def entity_required_configuration_keys( + field: ExtensionPointField, name: str +) -> frozenset[str] | None: + """The configuration members `name` requires, or None if unmodelled. + + A rule may subscript only these: a required member that is absent or + ill-typed is reported at `("configuration", member)`, which stands the + rule down. An optional member can be legitimately absent with no + problem reported, so a rule that subscripts one raises `KeyError` out + of a validator instead of returning a verdict. + """ + shape = _ENTITY_SHAPES.get(field, {}).get(canonical_name(field, name)) + return None if shape is None else shape.config_required + + def modelled_entities() -> frozenset[tuple[ExtensionPointField, str]]: """Every `(extension point, name)` with a shape validator.""" return frozenset((field, name) for field, shapes in _ENTITY_SHAPES.items() for name in shapes) @@ -670,6 +685,7 @@ def blocking_problems( "blocking_problems", "entity_configuration_keys", "entity_name", + "entity_required_configuration_keys", "modelled_entities", "validate_known_chunk_grid_metadata", "validate_known_codec_metadata", diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py index 6a65924839..0970fa620f 100644 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -10,6 +10,7 @@ from __future__ import annotations import pkgutil +from typing import TYPE_CHECKING import pytest @@ -32,6 +33,14 @@ ) from zarr_metadata.v3._shape import modelled_entities +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.model._validation import ValidationProblem + from zarr_metadata.rules._spec import ArraySpec +from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME + # Entities the package models but that carry no composition rules: their # canonical shape is the whole of what we can say about them. Listed by # hand, keyed by extension point, so that adding a codec is a deliberate @@ -169,3 +178,29 @@ def _uses_both(document: object) -> tuple[()]: return () assert _uses_both.requires == frozenset({"a", "b"}) + + +def test_error_entity_rule_reads_an_unmodelled_member() -> None: + with pytest.raises(ValueError, match="does not model"): + + @entity_rule(ZARR_V3_ARRAY, CODECS, GZIP_CODEC_NAME, reads=frozenset({"nosuchmember"})) + def _unmodelled_member( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArraySpec, + ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered + return () + + +def test_error_entity_rule_reads_an_optional_member() -> None: + # Only a required member is safe to subscript: an absent optional one is + # reported by nothing, so the rule would raise out of a validator. + with pytest.raises(ValueError, match="reads_optional"): + + @entity_rule(ZARR_V3_ARRAY, CODECS, BYTES_CODEC_NAME, reads=frozenset({"endian"})) + def _subscripts_an_optional_member( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArraySpec, + ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered + return () diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 11d17d61aa..36d0909056 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -645,3 +645,22 @@ def test_error_endian_message_names_the_shard_index_type() -> None: ) assert loc == ("codecs", 0, "configuration", "index_codecs", 0, "configuration", "endian") assert "uint64" in message + + +def test_error_a_malformed_must_understand_does_not_suppress_the_entity() -> None: + # `must_understand` is part of the envelope, not the configuration, so + # a bad one says nothing about whether the configuration is readable. + problems = validate_array_metadata_v3( + { + **BASE, + "codecs": ( + { + "name": "transpose", + "configuration": {"order": (2, 1, 0)}, + "must_understand": "yes", + }, + "bytes", + ), + } + ) + assert ("codecs", 0, "configuration", "order") in {problem.loc for problem in problems} From be410cb06c4fc546e151fb0dbc9eab60a0356848 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 12:53:18 +0200 Subject: [PATCH 016/107] refactor(zarr-metadata): model what array a chunk grid governs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three sites derived "the shape of the chunk this pipeline encodes" by hand — the document's grid, a shard's inner chunk shape, and the shard index — and each got a different subset of the reasoning right. Patching them one at a time produced three near-identical fixes and missed two more cases, so model the thing itself. `rules._chunk_grid` answers the question per dimension instead of per grid: a `GovernedShape` has one entry per axis, `None` where the chunks differ or the metadata cannot be read, and is itself `None` only when not even the rank is known. Following zarrs, a grid is read from its metadata together with the array shape it partitions (neither determines a grid alone), its rank is always available, and its extents are reported dimension by dimension rather than as one shape. Two cases the previous derivations could not express now work. A rectilinear grid whose chunk shapes are uniform pins the shard shape, in all three spellings the spec allows, while one uniform on a single axis is judged there and declines elsewhere. A shard index is judged against chunks-per-shard plus a trailing dimension of 2, as the spec derives it, instead of against nothing. Also extracts entity configuration access into rules._entity so the grid module can read a grid's configuration without importing the dispatcher. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.bugfix.md | 13 ++ .../src/zarr_metadata/rules/_chunk_grid.py | 176 ++++++++++++++++++ .../zarr_metadata/rules/_entities/sharding.py | 24 +-- .../src/zarr_metadata/rules/_entity.py | 55 ++++++ .../src/zarr_metadata/rules/_registry.py | 89 ++------- .../tests/rules/test_chunk_grid.py | 148 +++++++++++++++ 6 files changed, 425 insertions(+), 80 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entity.py create mode 100644 packages/zarr-metadata/tests/rules/test_chunk_grid.py diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md index a73e21c9b4..06455eb45c 100644 --- a/packages/zarr-metadata/changes/4379.bugfix.md +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -28,3 +28,16 @@ rules layer: configuration — counted as "the configuration is unreadable" and stood down every composition rule for that entity. Only a problem at the entity itself or at `configuration` as a whole does that now. + +The three derivations of "what array does this pipeline encode?" are now +one abstraction, `rules._chunk_grid`, which answers per dimension rather +than per grid. Following zarrs, a chunk grid is read from its metadata +*and* the array shape it partitions, its rank is always available, and its +extents are reported dimension by dimension. That resolves two more cases +the ad-hoc derivations could not express: a rectilinear grid whose chunk +shapes are uniform now pins the shard shape (so a shard that does not +divide it is reported, in all three spellings the spec allows), while a +grid uniform on one axis only is judged on that axis and declines on the +other; and a shard index is judged against its own derived shape — +chunks-per-shard plus a trailing dimension of 2 — rather than against no +shape at all. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py new file mode 100644 index 0000000000..295d7aa889 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py @@ -0,0 +1,176 @@ +"""What array a chunk grid governs. + +A codec pipeline encodes one chunk, so every rule about a pipeline — +transpose ranks, shard divisibility — needs the shape of that chunk. Three +different pieces of metadata answer that question, and this module is the +one place that reads them: + +- a document's `chunk_grid`, which partitions an array of `shape`; +- a `sharding_indexed` codec's `chunk_shape`, which is a regular grid over + the chunk the codec receives; +- that same codec's shard index, whose shape the spec derives from the + first two. + +Derived per dimension, not per grid +----------------------------------- +The answer is a `GovernedShape`: one entry per dimension, each the extent +every chunk has along it, or `None` where the chunks differ or the metadata +cannot be read. Only the whole result is `None`, and only when not even the +rank is known. + +That granularity is the point. A rectilinear grid has no single chunk +shape, but a *dimension* of one may still be uniform — `[[32, 32], [10, 22]]` +pins the first axis at 32 and says nothing about the second — and a shard +inside it can be judged on the axis that is pinned. A grid this package +cannot read at all still pins the rank, because every chunk of an array has +the array's rank whatever partitions it. Collapsing any of this to "shape +unknown" silently retires rules that had enough information to run. + +Prior art +--------- +zarrs models the same thing and is worth following: its `ChunkGrid` is +built from metadata *and* the array shape (`ChunkGrid::create(metadata, +array_shape)`) because neither alone determines a grid; its +`dimensionality()` is total rather than optional; and it reports +`chunk_edge_lengths(dimension)` per dimension for exactly the reason above. +Its codec chain carries a three-state `ChunkGridMapped` — `Array`, +`ChunkLocal` ("no global grid, but one per chunk"), `None` — keeping +"varies" distinct from "unknown"; a `GovernedShape` entry of `None` is the +per-dimension form of that distinction. + +- https://github.com/zarrs/zarrs/blob/main/zarrs_chunk_grid/src/lib.rs +- https://github.com/zarrs/zarrs/blob/main/zarrs_codec/src/lib.rs +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeAlias, cast + +from zarr_metadata.rules._entity import entity_configuration +from zarr_metadata.v3._extension_points import CHUNK_GRID +from zarr_metadata.v3._shape import entity_name +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID_NAME +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME + +if TYPE_CHECKING: + from collections.abc import Sequence + +GovernedShape: TypeAlias = "tuple[int | None, ...]" +"""One entry per dimension: the extent every chunk shares, or `None`. + +`None` in an entry means the chunks differ along that axis, or the value +is unusable. A `GovernedShape` always knows its rank; the absence of a +shape entirely is spelled `None` in place of the whole tuple. +""" + + +def _positive_int(value: object) -> int | None: + """`value` as a chunk extent, or None if it is not a usable one.""" + if isinstance(value, bool) or not isinstance(value, int): + return None + return value if value >= 1 else None + + +def rank_of(array_shape: object) -> int | None: + """The number of dimensions `array_shape` declares, if it declares any.""" + if not isinstance(array_shape, tuple): + return None + dimensions = cast("tuple[object, ...]", array_shape) + if not all(isinstance(v, int) and not isinstance(v, bool) for v in dimensions): + return None + return len(dimensions) + + +def uniform_shape(extents: Sequence[object]) -> GovernedShape: + """A regular grid's chunk shape: one declared extent per dimension. + + An unusable extent costs that dimension, never the rank — the grid's + own values rule owns the complaint about the value itself. + """ + return tuple(_positive_int(extent) for extent in extents) + + +def _rectilinear_extent(spec: object) -> int | None: + """The extent every chunk shares along one rectilinear dimension. + + A bare integer is a regular step, so it is that extent. An explicit + list is uniform only if every entry (run-length pairs expanded) names + the same size. Anything else varies, or cannot be read. + """ + bare = _positive_int(spec) + if bare is not None: + return bare + if not isinstance(spec, tuple): + return None + sizes: set[int] = set() + for item in cast("tuple[object, ...]", spec): + size = _positive_int(item) + if size is None and isinstance(item, tuple): + pair = cast("tuple[object, ...]", item) + size = _positive_int(pair[0]) if len(pair) == 2 else None + if _positive_int(pair[1]) is None: + return None + if size is None: + return None + sizes.add(size) + if len(sizes) != 1: + return None + return sizes.pop() + + +def governed_shape(grid: object, array_shape: object) -> GovernedShape | None: + """The shape of one chunk of `grid`, dimension by dimension. + + `grid` is a document's `chunk_grid` metadata and `array_shape` its + `shape`; a grid is not interpretable without the array it partitions, + and the array shape is what pins the rank when the grid itself cannot + be read. Answers None only when not even the rank is available. + """ + fallback = rank_of(array_shape) + name = entity_name(grid) + configuration = entity_configuration(CHUNK_GRID, grid) if name is not None else None + if configuration is not None: + if name == REGULAR_CHUNK_GRID_NAME: + extents = configuration.get("chunk_shape") + if isinstance(extents, tuple): + return uniform_shape(cast("tuple[object, ...]", extents)) + elif name == RECTILINEAR_CHUNK_GRID_NAME: + dimensions = configuration.get("chunk_shapes") + if isinstance(dimensions, tuple): + return tuple( + _rectilinear_extent(spec) for spec in cast("tuple[object, ...]", dimensions) + ) + return None if fallback is None else (None,) * fallback + + +def shard_index_shape(shard: GovernedShape | None, inner: Sequence[object]) -> GovernedShape: + """The shape of a shard's index array. + + The spec derives it from the two shapes around it: "The index is an + array with 64-bit unsigned integers with a shape that matches the + chunks per shard tuple with an appended dimension of size 2." Chunks + per shard needs both extents, so a dimension resolves only where the + shard and inner extents are both known and divide evenly; the rank is + one more than the inner chunk's, always. + """ + inner_shape = uniform_shape(inner) + # The trailing 2 is fixed by the spec, so it is known even when no + # chunk count is. + if shard is None or len(shard) != len(inner_shape): + return (*(None,) * len(inner_shape), 2) + counts = tuple( + outer // extent + if outer is not None and extent is not None and outer % extent == 0 + else None + for outer, extent in zip(shard, inner_shape, strict=True) + ) + return (*counts, 2) + + +__all__ = [ + "GovernedShape", + "governed_shape", + "rank_of", + "shard_index_shape", + "uniform_shape", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py index 97dce0c5a5..4921f050c7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py @@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, cast from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._chunk_grid import shard_index_shape, uniform_shape from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems from zarr_metadata.rules._registry import entity_rule, run_chain_rules from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec @@ -108,19 +109,20 @@ def inner_pipelines_are_pipelines( `codecs` chain starts from the inner chunk with the incoming data type; a nested shard or transpose inside it is therefore judged against the inner chunk, and its own transitions carry on from there. - The `index_codecs` chain encodes the shard index, a `uint64` array - whose shape this package does not compute. + The `index_codecs` chain encodes the shard index: a `uint64` array of + chunks-per-shard plus a trailing dimension of 2, derived by + `zarr_metadata.rules._chunk_grid.shard_index_shape`. """ - inner_shape = configuration["chunk_shape"] - if not isinstance(inner_shape, tuple) or not all( - isinstance(v, int) and not isinstance(v, bool) and v >= 1 - for v in cast("tuple[object, ...]", inner_shape) - ): + inner = configuration["chunk_shape"] + if not isinstance(inner, tuple): inner_start = NOTHING_KNOWN + index_start = NOTHING_KNOWN else: - # The inner pipeline encodes the inner chunk: same type as arrived - # here, shape of one inner chunk. - inner_start = incoming.with_shape(cast("tuple[int, ...]", inner_shape)) + extents = cast("tuple[object, ...]", inner) + # The inner chunk shape is a regular grid over the chunk this codec + # receives, and the index's shape follows from the two together. + inner_start = incoming.with_shape(uniform_shape(extents)) + index_start = ArraySpec(shard_index_shape(incoming.shape, extents), "uint64") problems: list[ValidationProblem] = [] for key in ("codecs", "index_codecs"): entries = configuration[key] @@ -132,7 +134,7 @@ def inner_pipelines_are_pipelines( # The index pipeline encodes the shard index, not the array: a # uint64 array of offsets and lengths, so e.g. the bytes codec # inside it still needs an endianness. - start = inner_start if key == "codecs" else ArraySpec(None, "uint64") + start = inner_start if key == "codecs" else index_start problems.extend(run_chain_rules(CODECS, sequence, document, (key,), start)) return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entity.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entity.py new file mode 100644 index 0000000000..955a186e3e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entity.py @@ -0,0 +1,55 @@ +"""Reading a named entity's configuration. + +An entity's `configuration` is only worth reading once its shape has been +vouched for, and there are two useful strictnesses. `entity_configuration` +is all-or-nothing, for callers that derive something from the whole +configuration (a spec transition). `run_entity_rules` wants the finer +per-member judgment and reaches for `configuration_mapping` plus the shape +verdict directly. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr_metadata.rules._engine import as_string_mapping +from zarr_metadata.v3._shape import blocking_problems, validate_known_entity_metadata + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._extension_points import ExtensionPointField + + +def entity_configuration(field: ExtensionPointField, value: object) -> Mapping[str, object] | None: + """`value`'s configuration if every modelled field is usable, else None. + + The all-or-nothing gate `propagate` needs: a spec transition reads the + configuration to compute what the next codec receives, so one unusable + member makes the whole outgoing spec a guess. `run_entity_rules` uses + the finer per-member gate instead. `unknown_key` problems do not make + an entity unusable; anything else does. + """ + verdict = validate_known_entity_metadata(field, value) + if verdict is None or len(blocking_problems(verdict)) != 0: + return None + return configuration_mapping(value) + + +def configuration_mapping(value: object) -> Mapping[str, object] | None: + """`value`'s configuration mapping, with no judgment of its contents.""" + mapping = as_string_mapping(value) + if mapping is None: + # Bare-string metadata is the canonical spelling for entities whose + # configuration is optional. Rules still need a real mapping to run + # against, especially when they judge a missing optional member. + return {} if isinstance(value, str) else None + if "configuration" not in mapping: + return {} + return as_string_mapping(mapping["configuration"]) + + +__all__ = [ + "configuration_mapping", + "entity_configuration", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py index 66c9b64075..d5af281517 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -12,12 +12,18 @@ from collections import defaultdict from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Final, cast +from typing import TYPE_CHECKING, Final, cast from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._engine import Rule, as_string_mapping +from zarr_metadata.rules._chunk_grid import governed_shape +from zarr_metadata.rules._engine import Rule +from zarr_metadata.rules._entity import configuration_mapping, entity_configuration from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec, propagate -from zarr_metadata.v3._extension_points import CHUNK_GRID, ExtensionPointField, canonical_name +from zarr_metadata.v3._extension_points import ExtensionPointField, canonical_name + +if TYPE_CHECKING: + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._shape import ( blocking_problems, entity_configuration_keys, @@ -252,7 +258,7 @@ def run_entity_rules( for problem in blocking if len(problem.loc) >= 2 and problem.loc[0] == "configuration" ) - configuration = _configuration_of(value) + configuration = configuration_mapping(value) if configuration is None: return () problems: list[ValidationProblem] = [] @@ -270,34 +276,6 @@ def run_entity_rules( return tuple(problems) -def entity_configuration(field: ExtensionPointField, value: object) -> Mapping[str, object] | None: - """`value`'s configuration if every modelled field is usable, else None. - - The all-or-nothing gate `propagate` needs: a spec transition reads the - configuration to compute what the next codec receives, so one unusable - member makes the whole outgoing spec a guess. `run_entity_rules` uses - the finer per-member gate instead. `unknown_key` problems do not make - an entity unusable; anything else does. - """ - verdict = validate_known_entity_metadata(field, value) - if verdict is None or len(blocking_problems(verdict)) != 0: - return None - return _configuration_of(value) - - -def _configuration_of(value: object) -> Mapping[str, object] | None: - """`value`'s configuration mapping, with no judgment of its contents.""" - mapping = as_string_mapping(value) - if mapping is None: - # Bare-string metadata is the canonical spelling for entities whose - # configuration is optional. Rules still need a real mapping to run - # against, especially when they judge a missing optional member. - return {} if isinstance(value, str) else None - if "configuration" not in mapping: - return {} - return as_string_mapping(mapping["configuration"]) - - def dispatch_field( field: ExtensionPointField, ) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: @@ -350,48 +328,21 @@ def run_chain_rules( return tuple(problems) -def _rank_only(shape: object) -> tuple[int | None, ...] | None: - """A shape of the document's rank with every extent undetermined.""" - if not isinstance(shape, tuple): - return None - dimensions = cast("tuple[object, ...]", shape) - if not all(isinstance(v, int) and not isinstance(v, bool) for v in dimensions): - return None - return (None,) * len(dimensions) - - def chain_initial_spec(document: Mapping[str, object]) -> ArraySpec: """The spec entering a document's top-level codec chain. - The array a chunk pipeline encodes is one chunk: extents from a - regular grid this package can read, data type from the document. - - When the extents are unavailable — a grid this package does not model, - a rectilinear grid whose chunks differ, or a regular grid with a - non-positive extent — the rank survives them. Every chunk of an array - has the array's rank, so `shape` becomes a tuple of `None` of that - length rather than `None`, and rank rules keep working while the - geometry rules stand down. (Geometry against a zero extent would be - noise on top of the grid's own complaint; a rank mismatch is a - separate fault and is still worth reporting.) + The array a chunk pipeline encodes is one chunk of the document's + chunk grid, so its shape is whatever that grid governs; see + `zarr_metadata.rules._chunk_grid`. """ - from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME - - grid = document.get("chunk_grid") - chunk_shape: tuple[int | None, ...] | None = None - if entity_name(grid) == REGULAR_CHUNK_GRID_NAME: - configuration = entity_configuration(CHUNK_GRID, grid) - extents = configuration.get("chunk_shape") if configuration is not None else None - if isinstance(extents, tuple): - values = cast("tuple[object, ...]", extents) - if all(isinstance(v, int) and not isinstance(v, bool) and v >= 1 for v in values): - chunk_shape = cast("tuple[int | None, ...]", values) - if chunk_shape is None: - chunk_shape = _rank_only(document.get("shape")) + chunk_shape = governed_shape(document.get("chunk_grid"), document.get("shape")) + # A metadata field is a name, or an object carrying one; anything else + # is not a data type this package can describe, and `entity_name` + # answers that question in one place rather than being re-derived here. data_type = document.get("data_type") - if not isinstance(data_type, (str, Mapping)): - data_type = None - return ArraySpec(chunk_shape, data_type) # type: ignore[arg-type] + if entity_name(data_type) is None: + return ArraySpec(chunk_shape, None) + return ArraySpec(chunk_shape, cast("ZarrV3MetadataFieldJSON", data_type)) __all__ = [ diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py new file mode 100644 index 0000000000..950e901745 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -0,0 +1,148 @@ +"""What array a chunk grid governs, and what follows from knowing it.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from zarr_metadata.rules import validate_array_metadata_v3 +from zarr_metadata.rules._chunk_grid import ( # pyright: ignore[reportPrivateUsage] + governed_shape, + shard_index_shape, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + +BASE: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (64, 64), + "data_type": "uint8", + "fill_value": 0, + "chunk_key_encoding": "default", +} +REGULAR: Mapping[str, object] = {"name": "regular", "configuration": {"chunk_shape": (32, 32)}} + + +def _rectilinear(chunk_shapes: object) -> Mapping[str, object]: + return { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": chunk_shapes}, + } + + +# The shard index is uint64, so its bytes codec needs an endianness; these +# tests are about geometry, not about that. +_INDEX_CODECS = ({"name": "bytes", "configuration": {"endian": "little"}},) + + +def _shard(inner: object, index: object = _INDEX_CODECS) -> Mapping[str, object]: + return { + "name": "sharding_indexed", + "configuration": {"chunk_shape": inner, "codecs": ("bytes",), "index_codecs": index}, + } + + +# (grid, array shape, the chunk shape it governs). A `None` entry is a +# dimension whose chunks differ or whose metadata cannot be read; a `None` +# result is a grid that does not even pin the rank. +GOVERNED: dict[str, tuple[object, object, object]] = { + "regular": (REGULAR, (64, 64), (32, 32)), + "regular-zero-extent-keeps-rank": ( + {"name": "regular", "configuration": {"chunk_shape": (0, 32)}}, + (64, 64), + (None, 32), + ), + "rectilinear-uniform": (_rectilinear(((32, 32), (32, 32))), (64, 64), (32, 32)), + "rectilinear-uniform-rle": (_rectilinear((((32, 2),), ((32, 2),))), (64, 64), (32, 32)), + "rectilinear-bare-int-is-a-regular-step": (_rectilinear((32, 32)), (64, 64), (32, 32)), + "rectilinear-mixed-resolves-per-dimension": ( + _rectilinear(((30, 34), (32, 32))), + (64, 64), + (None, 32), + ), + "unknown-grid-keeps-the-array-rank": ( + {"name": "mycorp.hilbert", "configuration": {"anything": 1}}, + (64, 64), + (None, None), + ), + "no-usable-array-shape": (None, "not a shape", None), +} + + +@pytest.mark.parametrize(("grid", "shape", "expected"), GOVERNED.values(), ids=list(GOVERNED)) +def test_governed_shape(grid: object, shape: object, expected: object) -> None: + assert governed_shape(grid, shape) == expected + + +def test_shard_index_shape_is_chunks_per_shard_plus_two() -> None: + # "a shape that matches the chunks per shard tuple with an appended + # dimension of size 2" — 128/32 = 4 along both axes. + assert shard_index_shape((128, 128), (32, 32)) == (4, 4, 2) + # The rank survives even when no extent does. + assert shard_index_shape(None, (32, 32)) == (None, None, 2) + assert shard_index_shape((None, 128), (32, 32)) == (None, 4, 2) + + +@pytest.mark.parametrize( + "chunk_shapes", + [((32, 32), (32, 32)), (((32, 2),), ((32, 2),)), (32, 32)], + ids=["explicit", "run-length", "bare-int"], +) +def test_error_shard_must_divide_a_uniform_rectilinear_grid(chunk_shapes: object) -> None: + # A rectilinear grid has no single chunk shape, but a uniform one pins + # every extent, and 7 divides neither. + problems = validate_array_metadata_v3( + {**BASE, "chunk_grid": _rectilinear(chunk_shapes), "codecs": (_shard((7, 7)),)} + ) + assert len(problems) == 2 + assert all("does not evenly divide" in problem.message for problem in problems) + + +def test_a_varying_dimension_declines_while_a_pinned_one_is_judged() -> None: + # chunk_shapes ((30, 34), (32, 32)): axis 0 varies, axis 1 is 32 + # everywhere. Only the axis that is knowable may be judged. + grid = _rectilinear(((30, 34), (32, 32))) + problems = validate_array_metadata_v3({**BASE, "chunk_grid": grid, "codecs": (_shard((7, 7)),)}) + assert [problem.loc for problem in problems] == [ + ("codecs", 0, "configuration", "chunk_shape", 1) + ] + # 32 divides the pinned axis, and axis 0 is unknown: nothing to report. + assert ( + validate_array_metadata_v3({**BASE, "chunk_grid": grid, "codecs": (_shard((7, 32)),)}) == () + ) + + +def test_error_index_codecs_are_judged_against_the_index_rank() -> None: + # The index of a 2-D shard is rank 3, so a rank-1 transpose is wrong. + index = ( + {"name": "transpose", "configuration": {"order": (0,)}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ) + problems = validate_array_metadata_v3( + {**BASE, "chunk_grid": REGULAR, "codecs": (_shard((8, 8), index=index),)} + ) + assert [problem.message for problem in problems] == [ + "order has 1 entries but the incoming array has 3 dimensions" + ] + + +def test_error_a_bad_inner_extent_costs_that_axis_and_nothing_else() -> None: + # The zero is reported, and the inner pipeline is still judged against + # the rank the inner chunk shape declares. + inner = _shard((0, 2)) + inner["configuration"] = { # type: ignore[index] + **inner["configuration"], # type: ignore[dict-item] + "codecs": ({"name": "transpose", "configuration": {"order": (0, 1, 2)}}, "bytes"), + } + messages = [ + problem.message + for problem in validate_array_metadata_v3( + {**BASE, "data_type": "uint16", "chunk_grid": REGULAR, "codecs": (inner,)} + ) + ] + assert any("positive chunk extent" in message for message in messages) + assert any("order has 3 entries" in message for message in messages) + assert any("endian is required" in message for message in messages) From ae61957dafb988031d4c2d8fa94a74d98c9ac645 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 13:28:37 +0200 Subject: [PATCH 017/107] refactor(zarr-metadata)!: carry the chunk grid, not a summary of it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A codec pipeline encodes one chunk, but the same pipeline encodes every chunk, so a chain rule is a statement about all of them. The propagated value said otherwise: `ArraySpec` held one chunk's shape, so an axis whose chunks differ collapsed to "unknown" and the rules that could have judged it stood down. `ArrayParts` carries the `ChunkGrid` instead. A grid holds its rank, its metadata as written, and per dimension the set of lengths that dimension's chunks take — singletons for a regular grid, `{30, 34}` for a rectilinear axis, `None` only where nothing can be read. Divisibility quantifies over that set and names the length that fails. This catches a case no previous shape could express: a rectilinear grid, a transpose moving its varying axis, and a sharding codec that has to divide what arrives. Inner extents of 15, 30 and 34 against chunks of 30 and 34 were all accepted before; only a common divisor passes now. Two things fall out of holding the grid rather than a projection of it. Sharding stops being a special case — a shard is a nested array, so its inner pipeline is built by the same constructor as the document's own — and a third-party grid is carried verbatim instead of being flattened at the first hop, so a future rule can read its own configuration. `data_type` becomes non-optional: the only documents that cannot supply one are documents the structural layer has already rejected, so rather than a half-populated value, a codec that can no longer be described receives nothing at all. `NOTHING_KNOWN` is gone in favour of `| None`, and rules test one guard instead of a field at a time. Named for what it is, after zarrs: a grid is built from metadata *and* the array shape it divides (neither determines a grid alone), its dimensionality is total, and its edge lengths are reported per dimension. `ArrayParts` avoids `chunk`, which would be singular, and `ArraySpec`, which is taken by zarr.core.array_spec for the runtime type. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.bugfix.md | 8 +- .../src/zarr_metadata/rules/_chunk_grid.py | 268 ++++++++++-------- .../rules/_entities/bytes_codec.py | 6 +- .../rules/_entities/cast_value.py | 6 +- .../src/zarr_metadata/rules/_entities/gzip.py | 4 +- .../rules/_entities/numpy_time.py | 4 +- .../rules/_entities/rectilinear_grid.py | 6 +- .../rules/_entities/regular_grid.py | 6 +- .../rules/_entities/scale_offset.py | 4 +- .../zarr_metadata/rules/_entities/sharding.py | 71 ++--- .../rules/_entities/struct_dtype.py | 12 +- .../rules/_entities/transpose.py | 41 +-- .../src/zarr_metadata/rules/_registry.py | 39 ++- .../src/zarr_metadata/rules/_spec.py | 108 ++++--- .../tests/rules/test_chunk_grid.py | 136 ++++++--- .../tests/rules/test_spec_propagation.py | 31 +- 16 files changed, 434 insertions(+), 316 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md index 06455eb45c..64d811f733 100644 --- a/packages/zarr-metadata/changes/4379.bugfix.md +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -31,7 +31,13 @@ rules layer: The three derivations of "what array does this pipeline encode?" are now one abstraction, `rules._chunk_grid`, which answers per dimension rather -than per grid. Following zarrs, a chunk grid is read from its metadata +than per grid, and plurally: what a codec is handed is `ArrayParts`, every +part of an array the pipeline will encode, carrying the chunk grid itself +rather than a summary of one. Every chain rule is a statement about all of +those parts — a shard's inner chunk shape must divide *every* chunk it +will see, which under a rectilinear grid is several different lengths, and +a transpose can move a varying axis into the position the shard has to +divide. Following zarrs, a chunk grid is read from its metadata *and* the array shape it partitions, its rank is always available, and its extents are reported dimension by dimension. That resolves two more cases the ad-hoc derivations could not express: a rectilinear grid whose chunk diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py index 295d7aa889..0b0ab3148a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py @@ -1,42 +1,38 @@ -"""What array a chunk grid governs. - -A codec pipeline encodes one chunk, so every rule about a pipeline — -transpose ranks, shard divisibility — needs the shape of that chunk. Three -different pieces of metadata answer that question, and this module is the -one place that reads them: - -- a document's `chunk_grid`, which partitions an array of `shape`; -- a `sharding_indexed` codec's `chunk_shape`, which is a regular grid over - the chunk the codec receives; -- that same codec's shard index, whose shape the spec derives from the - first two. - -Derived per dimension, not per grid ------------------------------------ -The answer is a `GovernedShape`: one entry per dimension, each the extent -every chunk has along it, or `None` where the chunks differ or the metadata -cannot be read. Only the whole result is `None`, and only when not even the -rank is known. - -That granularity is the point. A rectilinear grid has no single chunk -shape, but a *dimension* of one may still be uniform — `[[32, 32], [10, 22]]` -pins the first axis at 32 and says nothing about the second — and a shard -inside it can be judged on the axis that is pinned. A grid this package -cannot read at all still pins the rank, because every chunk of an array has -the array's rank whatever partitions it. Collapsing any of this to "shape -unknown" silently retires rules that had enough information to run. +"""How an array is divided, as far as this package can tell. + +A codec pipeline encodes one chunk, but the same pipeline encodes *every* +chunk, so a rule about a pipeline is a statement about all of them at +once: a shard's inner chunk shape must divide every chunk it will ever be +handed, not some representative one. `ChunkGrid` is what makes that +statement expressible. + +Three pieces of metadata divide an array, and this module is the one place +that reads them: a document's `chunk_grid`, a `sharding_indexed` codec's +`chunk_shape` (a regular grid over the chunk that codec receives), and +that codec's shard index, whose shape the spec derives from the other two. + +Per dimension, and plurally +--------------------------- +`extents` holds one entry per dimension: the set of lengths that +dimension's chunks take. A regular grid gives singletons; a rectilinear +grid gives `{30, 34}` on an axis whose chunks differ; `None` marks an axis +this package cannot read. `rank` survives even when no extent does, +because every chunk of an array has the array's rank whatever divides it. + +Collapsing any of that loses real judgments. A rectilinear grid uniform on +one axis still pins that axis, and a shard is judged there while declining +on the others. Prior art --------- -zarrs models the same thing and is worth following: its `ChunkGrid` is -built from metadata *and* the array shape (`ChunkGrid::create(metadata, -array_shape)`) because neither alone determines a grid; its -`dimensionality()` is total rather than optional; and it reports -`chunk_edge_lengths(dimension)` per dimension for exactly the reason above. -Its codec chain carries a three-state `ChunkGridMapped` — `Array`, -`ChunkLocal` ("no global grid, but one per chunk"), `None` — keeping -"varies" distinct from "unknown"; a `GovernedShape` entry of `None` is the -per-dimension form of that distinction. +zarrs builds its grid from metadata *and* the array shape +(`ChunkGrid::create(metadata, array_shape)`) because neither determines a +grid alone, keeps `dimensionality()` total rather than optional, and +reports `chunk_edge_lengths(dimension)` per dimension for the reason +above. Its codec chain distinguishes "no global grid, but one per chunk" +(`ChunkGridMapped::ChunkLocal`) from "nothing known" (`::None`); an +`extents` entry of `None` beside a known `rank` is the per-dimension form +of that distinction. - https://github.com/zarrs/zarrs/blob/main/zarrs_chunk_grid/src/lib.rs - https://github.com/zarrs/zarrs/blob/main/zarrs_codec/src/lib.rs @@ -44,6 +40,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING, TypeAlias, cast from zarr_metadata.rules._entity import entity_configuration @@ -55,23 +52,25 @@ if TYPE_CHECKING: from collections.abc import Sequence -GovernedShape: TypeAlias = "tuple[int | None, ...]" -"""One entry per dimension: the extent every chunk shares, or `None`. + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -`None` in an entry means the chunks differ along that axis, or the value -is unusable. A `GovernedShape` always knows its rank; the absence of a -shape entirely is spelled `None` in place of the whole tuple. +Extents: TypeAlias = "tuple[frozenset[int] | None, ...]" +"""One entry per dimension: the lengths that dimension's chunks take. + +A singleton is a uniform axis. `None` is an axis whose lengths this +package cannot determine — distinct from an empty set, which would claim +the axis has no chunks at all. """ def _positive_int(value: object) -> int | None: - """`value` as a chunk extent, or None if it is not a usable one.""" + """`value` as a chunk length, or None if it is not a usable one.""" if isinstance(value, bool) or not isinstance(value, int): return None return value if value >= 1 else None -def rank_of(array_shape: object) -> int | None: +def _rank_of(array_shape: object) -> int | None: """The number of dimensions `array_shape` declares, if it declares any.""" if not isinstance(array_shape, tuple): return None @@ -81,96 +80,143 @@ def rank_of(array_shape: object) -> int | None: return len(dimensions) -def uniform_shape(extents: Sequence[object]) -> GovernedShape: - """A regular grid's chunk shape: one declared extent per dimension. - - An unusable extent costs that dimension, never the rank — the grid's - own values rule owns the complaint about the value itself. - """ - return tuple(_positive_int(extent) for extent in extents) +def _uniform(lengths: Sequence[object]) -> Extents: + """Extents for a grid whose chunks are the same everywhere.""" + return tuple( + None if (length := _positive_int(value)) is None else frozenset({length}) + for value in lengths + ) -def _rectilinear_extent(spec: object) -> int | None: - """The extent every chunk shares along one rectilinear dimension. +def _rectilinear_axis(spec: object) -> frozenset[int] | None: + """The lengths one rectilinear dimension's chunks take. - A bare integer is a regular step, so it is that extent. An explicit - list is uniform only if every entry (run-length pairs expanded) names - the same size. Anything else varies, or cannot be read. + A bare integer is a regular step, so every chunk is that long. An + explicit list names them, with `[size, count]` pairs standing for + repeats; the distinct sizes are what any divisibility question needs. """ - bare = _positive_int(spec) - if bare is not None: - return bare + step = _positive_int(spec) + if step is not None: + return frozenset({step}) if not isinstance(spec, tuple): return None - sizes: set[int] = set() + lengths: set[int] = set() for item in cast("tuple[object, ...]", spec): size = _positive_int(item) if size is None and isinstance(item, tuple): pair = cast("tuple[object, ...]", item) - size = _positive_int(pair[0]) if len(pair) == 2 else None - if _positive_int(pair[1]) is None: + if len(pair) != 2 or _positive_int(pair[1]) is None: return None + size = _positive_int(pair[0]) if size is None: return None - sizes.add(size) - if len(sizes) != 1: - return None - return sizes.pop() + lengths.add(size) + return frozenset(lengths) if len(lengths) != 0 else None -def governed_shape(grid: object, array_shape: object) -> GovernedShape | None: - """The shape of one chunk of `grid`, dimension by dimension. +@dataclass(frozen=True, slots=True) +class ChunkGrid: + """The division of an array into the parts a codec pipeline encodes. - `grid` is a document's `chunk_grid` metadata and `array_shape` its - `shape`; a grid is not interpretable without the array it partitions, - and the array shape is what pins the rank when the grid itself cannot - be read. Answers None only when not even the rank is available. + `metadata` is the grid as the document spells it, kept so that a rule + for a grid this package does not model can still read its own + configuration. It is absent for a grid this package derived rather + than read — the regular grid a sharding codec imposes, or a transposed + grid — so nothing may validate it or report a location into it. """ - fallback = rank_of(array_shape) - name = entity_name(grid) - configuration = entity_configuration(CHUNK_GRID, grid) if name is not None else None - if configuration is not None: - if name == REGULAR_CHUNK_GRID_NAME: - extents = configuration.get("chunk_shape") - if isinstance(extents, tuple): - return uniform_shape(cast("tuple[object, ...]", extents)) - elif name == RECTILINEAR_CHUNK_GRID_NAME: - dimensions = configuration.get("chunk_shapes") - if isinstance(dimensions, tuple): - return tuple( - _rectilinear_extent(spec) for spec in cast("tuple[object, ...]", dimensions) - ) - return None if fallback is None else (None,) * fallback - - -def shard_index_shape(shard: GovernedShape | None, inner: Sequence[object]) -> GovernedShape: - """The shape of a shard's index array. + + rank: int | None + extents: Extents | None + metadata: ZarrV3MetadataFieldJSON | None = None + + @classmethod + def of(cls, grid: object, array_shape: object) -> ChunkGrid: + """The grid `grid` describes over an array of `array_shape`. + + A grid is not interpretable without the array it divides: the + array shape is what pins the rank when the grid itself cannot be + read, which is the case for every third-party grid. + """ + name = entity_name(grid) + configuration = entity_configuration(CHUNK_GRID, grid) if name is not None else None + metadata = cast("ZarrV3MetadataFieldJSON", grid) if name is not None else None + if configuration is not None: + if name == REGULAR_CHUNK_GRID_NAME: + lengths = configuration.get("chunk_shape") + if isinstance(lengths, tuple): + uniform = _uniform(cast("tuple[object, ...]", lengths)) + return cls(len(uniform), uniform, metadata) + elif name == RECTILINEAR_CHUNK_GRID_NAME: + axes = configuration.get("chunk_shapes") + if isinstance(axes, tuple): + varying = tuple( + _rectilinear_axis(axis) for axis in cast("tuple[object, ...]", axes) + ) + return cls(len(varying), varying, metadata) + rank = _rank_of(array_shape) + return cls(rank, None if rank is None else (None,) * rank, metadata) + + @classmethod + def derived(cls, extents: Extents) -> ChunkGrid: + """A grid this package computed rather than read from a document.""" + return cls(len(extents), extents) + + @classmethod + def regular(cls, lengths: Sequence[object]) -> ChunkGrid: + """The regular grid a sharding codec's `chunk_shape` imposes.""" + return cls.derived(_uniform(lengths)) + + def permuted(self, order: Sequence[int]) -> ChunkGrid: + """This grid with its dimensions reordered by `order`. + + A transposed grid is still a grid — permuting a regular one gives + a regular one — but it is no longer the grid the document wrote, + so the metadata does not survive the trip. + """ + if self.extents is None or len(order) != len(self.extents): + return ChunkGrid(self.rank, None) + return ChunkGrid.derived(tuple(self.extents[axis] for axis in order)) + + def axis(self, dimension: int) -> frozenset[int] | None: + """The lengths `dimension`'s chunks take, or None if undetermined.""" + if self.extents is None or dimension >= len(self.extents): + return None + return self.extents[dimension] + + +UNKNOWN_GRID: ChunkGrid = ChunkGrid(None, None) +"""A grid nothing is known about — not even how many dimensions it has.""" + + +def shard_index_grid(shard: ChunkGrid, inner: Sequence[object]) -> ChunkGrid: + """The grid of a shard's index array. The spec derives it from the two shapes around it: "The index is an array with 64-bit unsigned integers with a shape that matches the - chunks per shard tuple with an appended dimension of size 2." Chunks - per shard needs both extents, so a dimension resolves only where the - shard and inner extents are both known and divide evenly; the rank is - one more than the inner chunk's, always. + chunks per shard tuple with an appended dimension of size 2." The + index is one array rather than a divided one, so each axis holds a + single length — except that under a rectilinear grid the shard itself + varies, so the chunk count varies with it and the axis holds every + value it takes. """ - inner_shape = uniform_shape(inner) - # The trailing 2 is fixed by the spec, so it is known even when no - # chunk count is. - if shard is None or len(shard) != len(inner_shape): - return (*(None,) * len(inner_shape), 2) - counts = tuple( - outer // extent - if outer is not None and extent is not None and outer % extent == 0 - else None - for outer, extent in zip(shard, inner_shape, strict=True) - ) - return (*counts, 2) + inner_extents = _uniform(inner) + trailing: frozenset[int] | None = frozenset({2}) + if shard.extents is None or len(shard.extents) != len(inner_extents): + return ChunkGrid.derived((*(None,) * len(inner_extents), trailing)) + counts: list[frozenset[int] | None] = [] + for lengths, divisor in zip(shard.extents, inner_extents, strict=True): + if lengths is None or divisor is None: + counts.append(None) + continue + step = next(iter(divisor)) + quotients = {length // step for length in lengths if length % step == 0} + counts.append(frozenset(quotients) if len(quotients) == len(lengths) else None) + return ChunkGrid.derived((*counts, trailing)) __all__ = [ - "GovernedShape", - "governed_shape", - "rank_of", - "shard_index_shape", - "uniform_shape", + "UNKNOWN_GRID", + "ChunkGrid", + "Extents", + "shard_index_grid", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py index 3e5624beca..0c06b5e468 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py @@ -14,16 +14,16 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts _ARRAY_V3 = "zarr_v3_array" @entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME, reads_optional=frozenset({"endian"})) def data_type_has_a_raw_byte_representation( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: - if incoming.data_type is None: + if incoming is None: return () shape_verdict = validate_known_entity_metadata(DATA_TYPE, incoming.data_type) if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py index 24c1982cbe..8e328392ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py @@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, cast from zarr_metadata.rules._registry import entity_rule, run_entity_rules -from zarr_metadata.rules._spec import ArraySpec, spec_transition +from zarr_metadata.rules._spec import ArrayParts, spec_transition from zarr_metadata.v3._extension_points import CODECS, DATA_TYPE from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME @@ -30,14 +30,14 @@ @entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME, reads=frozenset({"data_type"})) def target_data_type_obeys_its_rules( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """A cast target obeys the same entity rules as a top-level data type.""" return run_entity_rules(DATA_TYPE, configuration["data_type"], document, ("data_type",)) @spec_transition(CAST_VALUE_CODEC_NAME) -def cast_data_type(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: +def cast_data_type(configuration: Mapping[str, object], incoming: ArrayParts) -> ArrayParts: """The outgoing type is the configured target.""" target = cast("ZarrV3MetadataFieldJSON", configuration["data_type"]) return incoming.with_data_type(target) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py index d73f3fb9fc..b26744dbfc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py @@ -12,14 +12,14 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts _ARRAY_V3 = "zarr_v3_array" @entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME, reads=frozenset({"level"})) def level_is_in_range( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: level = cast("int", configuration["level"]) if 0 <= level <= 9: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py index 934b488050..c18a179315 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts _ARRAY_V3 = "zarr_v3_array" _MAX_SCALE_FACTOR = 2**31 - 1 @@ -21,7 +21,7 @@ def _scale_factor_is_in_range( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: scale_factor = cast("int", configuration["scale_factor"]) if 1 <= scale_factor <= _MAX_SCALE_FACTOR: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py index af41ab4e44..c6a70eb58a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.rules._spec import ArraySpec, Sequence + from zarr_metadata.rules._spec import ArrayParts, Sequence _ARRAY_V3 = "zarr_v3_array" @@ -46,7 +46,7 @@ def _expanded_extent(spec: Sequence[object]) -> int | None: @entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shapes"})) def chunk_extents_are_positive( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """Every chunk extent, bare or run-length encoded, must be positive.""" chunk_shapes = cast("tuple[object, ...]", configuration["chunk_shapes"]) @@ -93,7 +93,7 @@ def chunk_extents_are_positive( reads=frozenset({"chunk_shapes"}), ) def tiles_the_array( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must sum to that extent. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py index 51c8d8db56..e1140ddd9f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py @@ -12,14 +12,14 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts _ARRAY_V3 = "zarr_v3_array" @entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shape"})) def chunk_extents_are_positive( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """Every chunk extent must be at least one element. @@ -47,7 +47,7 @@ def chunk_extents_are_positive( reads=frozenset({"chunk_shape"}), ) def chunks_every_dimension( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" shape = document["shape"] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py index 631b21225c..734f418a3e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py @@ -35,10 +35,10 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts @spec_transition(SCALE_OFFSET_CODEC_NAME) -def preserves_the_array(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: +def preserves_the_array(configuration: Mapping[str, object], incoming: ArrayParts) -> ArrayParts: """Element-wise arithmetic in the input type: same shape, same type.""" return incoming diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py index 4921f050c7..14c91720dd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py @@ -23,10 +23,10 @@ from typing import TYPE_CHECKING, cast from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._chunk_grid import shard_index_shape, uniform_shape +from zarr_metadata.rules._chunk_grid import ChunkGrid, shard_index_grid from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems from zarr_metadata.rules._registry import entity_rule, run_chain_rules -from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec +from zarr_metadata.rules._spec import ArrayParts from zarr_metadata.v3._extension_points import CODECS from zarr_metadata.v3._shape import entity_name from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME @@ -48,7 +48,7 @@ @entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE) def inner_chunk_extents_are_positive( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) return tuple( @@ -64,43 +64,48 @@ def inner_chunk_extents_are_positive( @entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE) def inner_chunks_tile_the_incoming_array( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: - """The inner chunk must rank-match and evenly divide the array it receives. + """The inner chunk must rank-match and divide every chunk it receives. - Declines when the incoming array is unknown entirely — an unclassified - codec upstream — rather than guessing. A known rank with unknown - extents (a chunk grid this package cannot read) still supports the - rank check; only the divisibility check needs the extents. + One sharding configuration encodes every chunk, so its inner shape has + to divide all of them. Under a rectilinear grid an axis has several + lengths and the inner extent must divide each; an axis whose lengths + are unknown declines while the others are still judged. """ - if incoming.shape is None: + if incoming is None or incoming.grid.rank is None: return () - outer = incoming.shape inner = cast("tuple[int, ...]", configuration["chunk_shape"]) - if len(inner) != len(outer): + if len(inner) != incoming.grid.rank: return ( ValidationProblem( ("chunk_shape",), f"chunk_shape has {len(inner)} entries but the incoming array has " - f"{len(outer)} dimensions", + f"{incoming.grid.rank} dimensions", "invalid_value", ), ) - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"inner chunk extent {inner_extent} does not evenly divide the " - f"incoming extent {outer_extent}", - "invalid_value", - ) - for position, (outer_extent, inner_extent) in enumerate(zip(outer, inner, strict=True)) - if outer_extent is not None and inner_extent >= 1 and outer_extent % inner_extent != 0 - ) + problems: list[ValidationProblem] = [] + for position, extent in enumerate(inner): + lengths = incoming.grid.axis(position) + if lengths is None or extent < 1: + continue + indivisible = sorted(length for length in lengths if length % extent != 0) + if len(indivisible) != 0: + problems.append( + ValidationProblem( + ("chunk_shape", position), + f"inner chunk extent {extent} does not evenly divide the incoming " + f"extent {indivisible[0]}", + "invalid_value", + ) + ) + return tuple(problems) @entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_PIPELINES) def inner_pipelines_are_pipelines( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """`codecs` and `index_codecs` obey the pipeline rules, recursively. @@ -114,15 +119,17 @@ def inner_pipelines_are_pipelines( `zarr_metadata.rules._chunk_grid.shard_index_shape`. """ inner = configuration["chunk_shape"] - if not isinstance(inner, tuple): - inner_start = NOTHING_KNOWN - index_start = NOTHING_KNOWN + if not isinstance(inner, tuple) or incoming is None: + inner_start: ArrayParts | None = None + index_start: ArrayParts | None = None else: extents = cast("tuple[object, ...]", inner) - # The inner chunk shape is a regular grid over the chunk this codec - # receives, and the index's shape follows from the two together. - inner_start = incoming.with_shape(uniform_shape(extents)) - index_start = ArraySpec(shard_index_shape(incoming.shape, extents), "uint64") + # A shard is a nested array: `chunk_shape` is a regular grid over + # the chunk this codec receives, so the inner pipeline is built + # exactly like the document's own, and the index's grid follows + # from the two together. + inner_start = incoming.with_grid(ChunkGrid.regular(extents)) + index_start = ArrayParts(shard_index_grid(incoming.grid, extents), "uint64") problems: list[ValidationProblem] = [] for key in ("codecs", "index_codecs"): entries = configuration[key] @@ -141,7 +148,7 @@ def inner_pipelines_are_pipelines( @entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_INDEX_CODECS) def index_codecs_have_fixed_encoded_size( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """The shard index must have an encoded size derivable from metadata.""" entries = cast("tuple[object, ...]", configuration["index_codecs"]) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py index 2de41bc63f..f9e0650aab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py @@ -25,14 +25,14 @@ from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME if TYPE_CHECKING: - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts _ARRAY_V3 = "zarr_v3_array" @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_data_types_obey_their_rules( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """Apply every known data type's rules inside struct fields, recursively.""" problems: list[ValidationProblem] = [] @@ -71,7 +71,7 @@ def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str], @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def fields_are_non_empty( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: fields = cast("tuple[object, ...]", configuration["fields"]) if len(fields) != 0: @@ -81,7 +81,7 @@ def fields_are_non_empty( @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_data_types_are_fixed_size( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: fields = cast("tuple[object, ...]", configuration["fields"]) problems: list[ValidationProblem] = [] @@ -102,7 +102,7 @@ def field_data_types_are_fixed_size( @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_names_are_non_empty( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """A struct field must be addressable, so its name cannot be empty.""" return tuple( @@ -116,7 +116,7 @@ def field_names_are_non_empty( @entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) def field_names_are_unique( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """Duplicate field names make a fill value's per-field mapping ambiguous.""" seen: dict[str, int] = {} diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py index 7e97a9d31a..ee20e515e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py @@ -5,8 +5,9 @@ from typing import TYPE_CHECKING, cast from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._chunk_grid import ChunkGrid from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.rules._spec import ArraySpec, spec_transition +from zarr_metadata.rules._spec import ArrayParts, spec_transition from zarr_metadata.v3._extension_points import CODECS from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME @@ -17,23 +18,23 @@ @spec_transition(TRANSPOSE_CODEC_NAME) -def permute_shape(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec: - """The outgoing shape is the incoming shape permuted by `order`. +def permute_grid(configuration: Mapping[str, object], incoming: ArrayParts) -> ArrayParts: + """The outgoing grid is the incoming one with its axes reordered. - Declines (shape None) when the order is not a permutation of the - incoming rank: the rules below report that, and any shape derived - from a bad order would be a guess. + A transposed grid is still a grid, so the parts survive the codec with + their lengths permuted. An order that is not a permutation of the rank + yields a grid of unknown extents — the rules below report the order + itself, and extents derived from a bad order would be a guess. """ order = cast("tuple[int, ...]", configuration["order"]) - shape = incoming.shape - if shape is None or sorted(order) != list(range(len(shape))): - return incoming.with_shape(None) - return incoming.with_shape(tuple(shape[axis] for axis in order)) + if sorted(order) != list(range(len(order))): + return incoming.with_grid(ChunkGrid(incoming.grid.rank, None)) + return incoming.with_grid(incoming.grid.permuted(order)) @entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) def order_is_a_permutation( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """`order` must be a permutation of its own indices. @@ -54,25 +55,25 @@ def order_is_a_permutation( @entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) def order_matches_incoming_rank( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec + configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. - Judged against the *incoming* spec, not the document's `shape`: inside - a shard the incoming array is the inner chunk, and after another - transpose it is that transpose's output. Declines when the incoming - shape is unknown. + Judged against what actually reaches this codec, not the document's + `shape`: inside a shard that is the inner chunk, and after another + transpose it is that transpose's output. Declines when the rank is + unknown. """ - if incoming.shape is None: + rank = incoming.grid.rank if incoming is not None else None + if rank is None: return () order = cast("tuple[int, ...]", configuration["order"]) - if len(order) == len(incoming.shape): + if len(order) == rank: return () return ( ValidationProblem( ("order",), - f"order has {len(order)} entries but the incoming array has " - f"{len(incoming.shape)} dimensions", + f"order has {len(order)} entries but the incoming array has {rank} dimensions", "invalid_value", ), ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py index d5af281517..104a1dc3db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -15,10 +15,10 @@ from typing import TYPE_CHECKING, Final, cast from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._chunk_grid import governed_shape +from zarr_metadata.rules._chunk_grid import ChunkGrid from zarr_metadata.rules._engine import Rule from zarr_metadata.rules._entity import configuration_mapping, entity_configuration -from zarr_metadata.rules._spec import NOTHING_KNOWN, ArraySpec, propagate +from zarr_metadata.rules._spec import ArrayParts, propagate from zarr_metadata.v3._extension_points import ExtensionPointField, canonical_name if TYPE_CHECKING: @@ -34,16 +34,16 @@ ) EntityCheck = Callable[ - [Mapping[str, object], Mapping[str, object], "ArraySpec"], + [Mapping[str, object], Mapping[str, object], "ArrayParts | None"], "tuple[ValidationProblem, ...]", ] """An entity rule's check: `(configuration, document, incoming)` in, problems out. -`incoming` is the `ArraySpec` the entity receives — for a codec, the -array as transformed by every codec before it in the chain. Fields this -package cannot determine are `None`; a caller with no chain context -passes `NOTHING_KNOWN`. Rules that need a field test it `is None` and -decline; rules that do not simply ignore the spec. +`incoming` is what the entity receives — for a codec, the array parts as +transformed by every codec before it in the chain, or `None` where this +package can no longer say. A caller with no chain context passes `None`. +Rules that need it test for `None` and decline; rules that do not simply +ignore it. Problems carry locations relative to the entity's `configuration`; the dispatcher re-bases them onto the entity's position in the document. @@ -227,7 +227,7 @@ def run_entity_rules( value: object, document: Mapping[str, object], loc: tuple[str | int, ...], - incoming: ArraySpec = NOTHING_KNOWN, + incoming: ArrayParts | None = None, ) -> tuple[ValidationProblem, ...]: """Run the rules registered for whatever entity `value` names. @@ -311,7 +311,7 @@ def run_chain_rules( codecs: Sequence[object], document: Mapping[str, object], loc: tuple[str | int, ...], - initial: ArraySpec, + initial: ArrayParts | None, ) -> tuple[ValidationProblem, ...]: """Run entity rules over a codec chain, propagating the array spec. @@ -328,21 +328,18 @@ def run_chain_rules( return tuple(problems) -def chain_initial_spec(document: Mapping[str, object]) -> ArraySpec: - """The spec entering a document's top-level codec chain. +def chain_initial_spec(document: Mapping[str, object]) -> ArrayParts | None: + """What enters a document's top-level codec chain. - The array a chunk pipeline encodes is one chunk of the document's - chunk grid, so its shape is whatever that grid governs; see - `zarr_metadata.rules._chunk_grid`. + The parts a chunk pipeline encodes are the chunks of the document's + chunk grid. A document whose `data_type` is not a metadata field has + already been rejected structurally, so there is nothing to describe. """ - chunk_shape = governed_shape(document.get("chunk_grid"), document.get("shape")) - # A metadata field is a name, or an object carrying one; anything else - # is not a data type this package can describe, and `entity_name` - # answers that question in one place rather than being re-derived here. data_type = document.get("data_type") if entity_name(data_type) is None: - return ArraySpec(chunk_shape, None) - return ArraySpec(chunk_shape, cast("ZarrV3MetadataFieldJSON", data_type)) + return None + grid = ChunkGrid.of(document.get("chunk_grid"), document.get("shape")) + return ArrayParts(grid, cast("ZarrV3MetadataFieldJSON", data_type)) __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py index 9fd60ef0c9..9127ae89a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py @@ -6,12 +6,12 @@ `cast_value` changes the data type, and a `sharding_indexed` codec that follows either one sees the transformed array. -`ArraySpec` is the array a codec receives; `propagate` walks a chain -handing each codec its incoming spec. A field is `None` once this package -can no longer determine it. An unknown codec might change anything, so -every codec after one receives `NOTHING_KNOWN` and rules that need a -field decline rather than guess. Shape stops at the array->bytes -boundary; the data type carries through. +`ArrayParts` is every part of an array a codec will be handed, together +with their element type; `propagate` walks a chain handing each codec what +reaches it. There is no half-populated value: a codec receives `None` once +this package can no longer say what it operates on. An unknown codec might +change anything, so everything after one receives `None`, and so does +everything after the array->bytes boundary, where there is no array left. Transitions are registered per array->array codec, next to that codec's rules, via `spec_transition`. A modelled codec with no transition is @@ -24,6 +24,7 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Final +from zarr_metadata.rules._chunk_grid import ChunkGrid # noqa: TC001 from zarr_metadata.v3._extension_points import CODECS, canonical_name from zarr_metadata.v3._shape import entity_name from zarr_metadata.v3.codec.kind import codec_kind_of_name @@ -33,44 +34,39 @@ @dataclass(frozen=True, slots=True) -class ArraySpec: - """The array a codec receives; a field is `None` when undetermined. - - `shape` is `None` only when there is no array left to describe (past - the array->bytes boundary) or when nothing about it can be determined. - An individual *extent* may be `None` while the rank is known: every - chunk of an array has the array's rank, whatever the chunk grid, so a - grid this package cannot read still pins `len(shape)`. Rules that need - a rank may use one; rules that need an extent test it for `None`. - - `data_type` is the metadata-field value verbatim (a bare name or a - name/configuration object) because rules compare it by name. +class ArrayParts: + """Every part of an array a codec will be handed, and their type. + + The parts an array is divided into, not the fields of its metadata. + Plural deliberately: one pipeline encodes every chunk, so a rule about + it quantifies over all of them — a shard's inner chunk shape must + divide *every* chunk, which under a rectilinear grid is several + different lengths. + + `data_type` is never absent. The only documents that cannot supply one + are documents the structural layer has already rejected, so "a real + array whose type we do not know" is not a state worth modelling; when + nothing is known, there are no `ArrayParts` at all. It is the + metadata-field value verbatim, because rules compare it by name. """ - shape: tuple[int | None, ...] | None - data_type: ZarrV3MetadataFieldJSON | None + grid: ChunkGrid + data_type: ZarrV3MetadataFieldJSON - def with_shape(self, shape: tuple[int | None, ...] | None) -> ArraySpec: - return replace(self, shape=shape) + def with_grid(self, grid: ChunkGrid) -> ArrayParts: + return replace(self, grid=grid) - def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArraySpec: + def with_data_type(self, data_type: ZarrV3MetadataFieldJSON) -> ArrayParts: return replace(self, data_type=data_type) -NOTHING_KNOWN: Final = ArraySpec(None, None) -"""The spec past a point where nothing about the array can be determined. +SpecTransition = Callable[[Mapping[str, object], "ArrayParts"], "ArrayParts | None"] +"""How one codec transforms what it receives. -Compare by equality: a spec can arrive here field by field and is then -equal to this constant without being it. -""" - - -SpecTransition = Callable[[Mapping[str, object], ArraySpec], ArraySpec] -"""How one codec transforms the spec it receives. - -Takes the codec's (shape-valid) configuration and the incoming spec, and -returns the outgoing one. A transition must never raise on the values the -shape validator admits; a field it cannot determine becomes `None`. +Takes the codec's (shape-valid) configuration and the incoming parts, and +returns what the next codec sees, or `None` when this codec leaves nothing +determinable. A transition must never raise on the values the shape +validator admits. """ _TRANSITIONS: Final[dict[str, SpecTransition]] = {} @@ -106,42 +102,42 @@ def transitions_registered() -> frozenset[str]: def propagate( codecs: Sequence[object], - initial: ArraySpec, + initial: ArrayParts | None, configuration_of: Callable[[object], Mapping[str, object] | None], -) -> Iterator[tuple[int, object, ArraySpec]]: - """Yield `(index, codec, incoming_spec)` for each codec in the chain. +) -> Iterator[tuple[int, object, ArrayParts | None]]: + """Yield `(index, codec, incoming)` for each codec in the chain. - `incoming_spec` is `NOTHING_KNOWN` once propagation has stopped: after - an unknown codec, after a known codec whose configuration is not - shape-valid, or after a codec this package has no transition for. + `incoming` is `None` once propagation has stopped: after an unknown + codec, after a known codec whose configuration is not shape-valid, + after a codec this package has no transition for, and after the + array->bytes boundary, where there is no array to describe. `configuration_of` resolves a codec entry to its usable configuration (`entity_configuration` in practice; injected to keep this module free of the registry). """ - spec = initial + parts = initial for index, codec in enumerate(codecs): - yield index, codec, spec - if spec == NOTHING_KNOWN: + yield index, codec, parts + if parts is None: continue name = entity_name(codec) kind = codec_kind_of_name(name) if name is not None else None - if kind is None: - spec = NOTHING_KNOWN - elif kind == "array_array": + if kind == "array_array": transition = _TRANSITIONS.get(canonical_name(CODECS, name or "")) configuration = configuration_of(codec) - if transition is None or configuration is None: - spec = NOTHING_KNOWN - else: - spec = transition(configuration, spec) + parts = ( + None + if transition is None or configuration is None + else transition(configuration, parts) + ) else: - # array->bytes: the array is gone; bytes->bytes: never had one. - spec = spec.with_shape(None) + # An unknown codec may change anything; array->bytes consumes + # the array; bytes->bytes never had one. + parts = None __all__ = [ - "NOTHING_KNOWN", - "ArraySpec", + "ArrayParts", "SpecTransition", "propagate", "spec_transition", diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index 950e901745..3321300552 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -7,10 +7,7 @@ import pytest from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.rules._chunk_grid import ( # pyright: ignore[reportPrivateUsage] - governed_shape, - shard_index_shape, -) +from zarr_metadata.rules._chunk_grid import ChunkGrid, shard_index_grid if TYPE_CHECKING: from collections.abc import Mapping @@ -45,45 +42,87 @@ def _shard(inner: object, index: object = _INDEX_CODECS) -> Mapping[str, object] } -# (grid, array shape, the chunk shape it governs). A `None` entry is a -# dimension whose chunks differ or whose metadata cannot be read; a `None` -# result is a grid that does not even pin the rank. -GOVERNED: dict[str, tuple[object, object, object]] = { - "regular": (REGULAR, (64, 64), (32, 32)), - "regular-zero-extent-keeps-rank": ( +def _u(*lengths: int) -> tuple[frozenset[int], ...]: + return tuple(frozenset({length}) for length in lengths) + + +# (grid metadata, array shape, rank, the lengths each axis's chunks take). +# A `None` axis is one this package cannot read; a `None` extents is a grid +# that does not even pin the rank. +GRIDS: dict[str, tuple[object, object, object, object]] = { + "regular": (REGULAR, (64, 64), 2, _u(32, 32)), + "regular-zero-length-keeps-rank": ( {"name": "regular", "configuration": {"chunk_shape": (0, 32)}}, (64, 64), - (None, 32), + 2, + (None, frozenset({32})), ), - "rectilinear-uniform": (_rectilinear(((32, 32), (32, 32))), (64, 64), (32, 32)), - "rectilinear-uniform-rle": (_rectilinear((((32, 2),), ((32, 2),))), (64, 64), (32, 32)), - "rectilinear-bare-int-is-a-regular-step": (_rectilinear((32, 32)), (64, 64), (32, 32)), - "rectilinear-mixed-resolves-per-dimension": ( + "rectilinear-uniform": (_rectilinear(((32, 32), (32, 32))), (64, 64), 2, _u(32, 32)), + "rectilinear-uniform-rle": (_rectilinear((((32, 2),), ((32, 2),))), (64, 64), 2, _u(32, 32)), + "rectilinear-bare-int-is-a-regular-step": (_rectilinear((32, 32)), (64, 64), 2, _u(32, 32)), + "rectilinear-varying-axis-keeps-its-lengths": ( _rectilinear(((30, 34), (32, 32))), (64, 64), - (None, 32), + 2, + (frozenset({30, 34}), frozenset({32})), + ), + "rectilinear-rle-varying": ( + _rectilinear( + ((((30, 2)), ((34, 1))),), + ), + (64,), + 1, + (frozenset({30, 34}),), ), "unknown-grid-keeps-the-array-rank": ( {"name": "mycorp.hilbert", "configuration": {"anything": 1}}, (64, 64), + 2, (None, None), ), - "no-usable-array-shape": (None, "not a shape", None), + "no-usable-array-shape": (None, "not a shape", None, None), } -@pytest.mark.parametrize(("grid", "shape", "expected"), GOVERNED.values(), ids=list(GOVERNED)) -def test_governed_shape(grid: object, shape: object, expected: object) -> None: - assert governed_shape(grid, shape) == expected +@pytest.mark.parametrize(("grid", "shape", "rank", "extents"), GRIDS.values(), ids=list(GRIDS)) +def test_chunk_grid_of(grid: object, shape: object, rank: object, extents: object) -> None: + built = ChunkGrid.of(grid, shape) + assert built.rank == rank + assert built.extents == extents + + +def test_an_unmodelled_grid_is_carried_verbatim() -> None: + # A rule for a third-party grid can still read its own configuration. + grid = {"name": "mycorp.hilbert", "configuration": {"order": 3}} + assert ChunkGrid.of(grid, (64, 64)).metadata == grid + + +def test_a_derived_grid_carries_no_metadata() -> None: + # Nothing may validate a grid this package invented, or report a + # location into one, so it must not look like a document's grid. + assert ChunkGrid.regular((8, 8)).metadata is None + assert ChunkGrid.of(REGULAR, (64, 64)).permuted((1, 0)).metadata is None + + +def test_permuting_reorders_the_axes() -> None: + grid = ChunkGrid.of(_rectilinear(((30, 34), (32, 32))), (64, 64)) + assert grid.permuted((1, 0)).extents == (frozenset({32}), frozenset({30, 34})) + # An order that is not a permutation of the rank keeps the rank only. + assert grid.permuted((0, 1, 2)).extents is None + assert grid.permuted((0, 1, 2)).rank == 2 -def test_shard_index_shape_is_chunks_per_shard_plus_two() -> None: +def test_shard_index_grid_is_chunks_per_shard_plus_two() -> None: # "a shape that matches the chunks per shard tuple with an appended - # dimension of size 2" — 128/32 = 4 along both axes. - assert shard_index_shape((128, 128), (32, 32)) == (4, 4, 2) - # The rank survives even when no extent does. - assert shard_index_shape(None, (32, 32)) == (None, None, 2) - assert shard_index_shape((None, 128), (32, 32)) == (None, 4, 2) + # dimension of size 2" -- 128/32 = 4 along both axes. + assert shard_index_grid(ChunkGrid.regular((128, 128)), (32, 32)).extents == _u(4, 4, 2) + # A shard whose own extents vary makes the chunk count vary with it. + assert shard_index_grid(ChunkGrid.derived((frozenset({30, 60}),)), (15,)).extents == ( + frozenset({2, 4}), + frozenset({2}), + ) + # The trailing 2 is fixed by the spec, so it survives knowing nothing. + assert shard_index_grid(ChunkGrid(None, None), (32, 32)).extents == (None, None, frozenset({2})) @pytest.mark.parametrize( @@ -101,18 +140,47 @@ def test_error_shard_must_divide_a_uniform_rectilinear_grid(chunk_shapes: object assert all("does not evenly divide" in problem.message for problem in problems) -def test_a_varying_dimension_declines_while_a_pinned_one_is_judged() -> None: - # chunk_shapes ((30, 34), (32, 32)): axis 0 varies, axis 1 is 32 - # everywhere. Only the axis that is knowable may be judged. +def test_error_a_shard_must_divide_every_chunk_a_varying_axis_has() -> None: + # The case the per-grid summary could not express: dim 0's chunks are + # 30 and 34 long, dim 1's are all 32, and a transpose swaps them before + # the shard sees them. The inner extent must divide *every* length the + # axis takes, so only a common divisor of 30 and 34 will do. grid = _rectilinear(((30, 34), (32, 32))) - problems = validate_array_metadata_v3({**BASE, "chunk_grid": grid, "codecs": (_shard((7, 7)),)}) - assert [problem.loc for problem in problems] == [ - ("codecs", 0, "configuration", "chunk_shape", 1) + transpose = {"name": "transpose", "configuration": {"order": (1, 0)}} + + def verdict(inner: object) -> list[object]: + return [ + problem.message + for problem in validate_array_metadata_v3( + {**BASE, "chunk_grid": grid, "codecs": (transpose, _shard(inner))} + ) + ] + + assert verdict((16, 2)) == [] # 2 divides both 30 and 34 + assert verdict((16, 15)) == [ # 15 divides 30 but not 34 + "inner chunk extent 15 does not evenly divide the incoming extent 34" + ] + assert verdict((16, 30)) == [ # 30 divides itself but not 34 + "inner chunk extent 30 does not evenly divide the incoming extent 34" + ] + assert verdict((16, 34)) == [ # and the other way round + "inner chunk extent 34 does not evenly divide the incoming extent 30" ] - # 32 divides the pinned axis, and axis 0 is unknown: nothing to report. + + +def test_an_unreadable_axis_declines_while_its_neighbours_are_judged() -> None: + # A grid this package cannot read pins the rank and nothing else, so + # every axis declines; a rank mismatch is still caught. + grid = {"name": "mycorp.hilbert", "configuration": {"order": 3}} assert ( - validate_array_metadata_v3({**BASE, "chunk_grid": grid, "codecs": (_shard((7, 32)),)}) == () + validate_array_metadata_v3({**BASE, "chunk_grid": grid, "codecs": (_shard((7, 7)),)}) == () ) + problems = validate_array_metadata_v3( + {**BASE, "chunk_grid": grid, "codecs": (_shard((7, 7, 7)),)} + ) + assert [problem.message for problem in problems] == [ + "chunk_shape has 3 entries but the incoming array has 2 dimensions" + ] def test_error_index_codecs_are_judged_against_the_index_rank() -> None: diff --git a/packages/zarr-metadata/tests/rules/test_spec_propagation.py b/packages/zarr-metadata/tests/rules/test_spec_propagation.py index 90ef9a2a13..26456a016b 100644 --- a/packages/zarr-metadata/tests/rules/test_spec_propagation.py +++ b/packages/zarr-metadata/tests/rules/test_spec_propagation.py @@ -1,4 +1,4 @@ -"""Tests for array-spec propagation through a codec chain. +"""Tests for propagating an array's parts through a codec chain. The property under test: every codec is judged against the array it *receives*, which is the document's chunk only for the first codec in @@ -13,12 +13,8 @@ import pytest from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.rules._spec import ( - NOTHING_KNOWN, - ArraySpec, - propagate, - transitions_registered, -) +from zarr_metadata.rules._chunk_grid import ChunkGrid +from zarr_metadata.rules._spec import ArrayParts, propagate, transitions_registered from zarr_metadata.v3.codec.kind import ARRAY_ARRAY_CODEC_NAMES if TYPE_CHECKING: @@ -113,13 +109,14 @@ def test_propagate_yields_incoming_spec_per_codec() -> None: from zarr_metadata.v3._extension_points import CODECS chain = (_transpose(1, 0), "bytes", "crc32c") - start = ArraySpec((6, 4), "uint8") + start = ArrayParts(ChunkGrid.regular((6, 4)), "uint8") seen = list(propagate(chain, start, lambda c: entity_configuration(CODECS, c))) incoming = [spec for _, _, spec in seen] - assert incoming[0] == ArraySpec((6, 4), "uint8") # transpose receives the chunk - assert incoming[1] == ArraySpec((4, 6), "uint8") # bytes receives the transposed chunk - # past array->bytes: no array, so no shape; the type carries through - assert incoming[2] == ArraySpec(None, "uint8") + assert incoming[0] == ArrayParts(ChunkGrid.regular((6, 4)), "uint8") + # bytes receives the transposed chunk + assert incoming[1] == ArrayParts(ChunkGrid.regular((4, 6)), "uint8") + # past the array->bytes boundary there is no array to describe + assert incoming[2] is None def test_cast_value_changes_the_downstream_data_type() -> None: @@ -127,20 +124,20 @@ def test_cast_value_changes_the_downstream_data_type() -> None: from zarr_metadata.v3._extension_points import CODECS chain = ({"name": "cast_value", "configuration": {"data_type": "float32"}}, "bytes") - start = ArraySpec((6, 4), "uint8") + start = ArrayParts(ChunkGrid.regular((6, 4)), "uint8") seen = list(propagate(chain, start, lambda c: entity_configuration(CODECS, c))) - assert seen[1][2] == ArraySpec((6, 4), "float32") + assert seen[1][2] == ArrayParts(ChunkGrid.regular((6, 4)), "float32") def test_unknown_codec_yields_nothing_known() -> None: from zarr_metadata.rules._registry import entity_configuration from zarr_metadata.v3._extension_points import CODECS - start = ArraySpec((6, 4), "uint8") + start = ArrayParts(ChunkGrid.regular((6, 4)), "uint8") seen = list( propagate(({"name": "zfpy"}, "bytes"), start, lambda c: entity_configuration(CODECS, c)) ) - assert seen[1][2] is NOTHING_KNOWN + assert seen[1][2] is None def test_every_array_array_codec_registers_a_transition() -> None: @@ -157,5 +154,5 @@ def test_error_transition_for_a_non_array_array_codec() -> None: with pytest.raises(ValueError, match="only array->array codecs"): @spec_transition("gzip") - def _nope(configuration: object, incoming: ArraySpec) -> ArraySpec: # pragma: no cover + def _nope(configuration: object, incoming: ArrayParts) -> ArrayParts: # pragma: no cover return incoming From 24da793acb679fcdfa78d90f40b337975baee6e3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 13:36:16 +0200 Subject: [PATCH 018/107] test(zarr-metadata): draw codec chains from the codec types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `st.from_type` resolves the codec TypedDicts unaided — ReadOnly, closed, NotRequired and Literal all work — once one strategy is registered for the recursive JSONValue alias, without which sharding_indexed, cast_value and scale_offset cannot be resolved at all. The existing totality test feeds arbitrary JSON to the document validators. That is the right guard for the structural layer and no guard for this one: a random object never names a codec, so across 5,000 examples it dispatched no entity rule. Drawing codecs from their own types and assembling the chain by kind takes documents reaching a chain rule from 0% to 94%; ordering is what does it, because a misordered chain is rejected before any other rule runs, and a flat list is misordered most of the time. Adds the property nothing covered: documents valid by construction, with inner chunk shapes drawn from the divisors of the extents they must divide, over both regular and rectilinear grids. Every recent fix made this layer stricter and under 5% of generated documents are valid, so the accept side had no generated coverage at all. Each strategy's reach is asserted, not assumed — the witnesses it must produce, and why the three chain rules it cannot reach are out of reach by construction. Verified by sabotage: stubbing out entity dispatch fails both reach tests, where previously the property suite stayed green. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.misc.md | 23 +++ .../zarr-metadata/tests/rules/strategies.py | 185 ++++++++++++++++++ .../tests/rules/test_chain_properties.py | 150 ++++++++++++++ 3 files changed, 358 insertions(+) create mode 100644 packages/zarr-metadata/changes/4379.misc.md create mode 100644 packages/zarr-metadata/tests/rules/strategies.py create mode 100644 packages/zarr-metadata/tests/rules/test_chain_properties.py diff --git a/packages/zarr-metadata/changes/4379.misc.md b/packages/zarr-metadata/changes/4379.misc.md new file mode 100644 index 0000000000..50a6691f84 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.misc.md @@ -0,0 +1,23 @@ +Property tests over codec chains drawn from the codec `TypedDict`s with +`hypothesis.strategies.from_type`, which needs one registered strategy for +the recursive `JSONValue` alias and then resolves `ReadOnly`, `closed`, +`NotRequired` and `Literal` unaided. + +The pre-existing totality test feeds arbitrary JSON to the document +validators; a random object never names a codec, so it dispatched no +entity rule at all. Drawing codecs from their own types and assembling the +chain by pipeline kind (`array->array`* `array->bytes` `bytes->bytes`*) +takes the proportion of documents reaching a chain rule from 0% to 94% — +ordering matters because a chain in the wrong order is rejected before any +other rule runs. + +Two properties are now covered that were not: that a chain of arbitrary +configurations always produces a verdict rather than raising, and that +documents built valid by construction — extents drawn, then inner chunk +shapes drawn from their divisors, across regular and rectilinear grids — +are accepted. The second guards the direction every recent fix pushed +against, since fewer than 5% of generated documents are valid. + +The reach of each strategy is asserted rather than assumed, so a change +that silently stops dispatching fails loudly instead of leaving a green +property run that tests nothing. diff --git a/packages/zarr-metadata/tests/rules/strategies.py b/packages/zarr-metadata/tests/rules/strategies.py new file mode 100644 index 0000000000..f3de60a893 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/strategies.py @@ -0,0 +1,185 @@ +"""Hypothesis strategies over codec chains, derived from the codec types. + +Two kinds of strategy, because the two halves of "is this validator right?" +need opposite inputs. + +`codec_chains` draws each codec with `st.from_type` over its own +`TypedDict`, so the values are structurally plausible and almost always +semantically wrong. That is what the rejection side wants. Crucially the +chain is assembled by *kind* — `array->array`* `array->bytes` +`bytes->bytes`* — because a chain in the wrong order is rejected by the +ordering rule before any other rule runs, and a flat list of random codecs +is in the wrong order most of the time. Ordering the chain takes the +proportion of documents that reach a chain rule from 39% to 94%. + +`valid_documents` builds documents that must be accepted, by construction: +extents are drawn, then inner chunk shapes are drawn from their divisors. +Nothing else here can test the accept side, because fewer than 5% of +`from_type` documents are valid. + +`st.from_type` handles `ReadOnly`, `closed=True`, `NotRequired` and +`Literal` unaided. It cannot resolve the recursive `JSONValue` alias, so +this module registers one for it; without that, every codec whose +configuration admits arbitrary JSON (`sharding_indexed`, `cast_value`, +`scale_offset`) fails to resolve. +""" + +from __future__ import annotations + +from math import gcd +from typing import TYPE_CHECKING, Any + +from hypothesis import strategies as st + +from zarr_metadata._common import JSONValue +from zarr_metadata.v3.codec.blosc import BloscCodecObject +from zarr_metadata.v3.codec.bytes import BytesCodecObject +from zarr_metadata.v3.codec.cast_value import CastValueCodecObject +from zarr_metadata.v3.codec.crc32c import Crc32cCodecObject +from zarr_metadata.v3.codec.gzip import GzipCodecObject +from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodecObject +from zarr_metadata.v3.codec.sharding_indexed import ShardingIndexedCodecObject +from zarr_metadata.v3.codec.transpose import TransposeCodecObject +from zarr_metadata.v3.codec.zstd import ZstdCodecObject + +if TYPE_CHECKING: + from collections.abc import Sequence + +JSON_VALUES = st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=8), + lambda children: ( + st.lists(children, max_size=3) | st.dictionaries(st.text(max_size=6), children, max_size=3) + ), + max_leaves=5, +) +st.register_type_strategy(JSONValue, JSON_VALUES) + +# The codec TypedDicts, by pipeline kind. Hand-written because there is no +# name-to-type table to derive it from; `test_strategies.py` asserts it +# covers every codec the package models. +ARRAY_ARRAY = (TransposeCodecObject, CastValueCodecObject, ScaleOffsetCodecObject) +ARRAY_BYTES = (BytesCodecObject, ShardingIndexedCodecObject) +BYTES_BYTES = (BloscCodecObject, Crc32cCodecObject, GzipCodecObject, ZstdCodecObject) + + +_LITTLE: dict[str, object] = {"name": "bytes", "configuration": {"endian": "little"}} + + +def _any_of(types: Sequence[type]) -> st.SearchStrategy[Any]: + return st.one_of(*[st.from_type(entry) for entry in types]) + + +def codec_chains() -> st.SearchStrategy[tuple[object, ...]]: + """Chains in the shape the spec requires, with arbitrary configurations.""" + return st.tuples( + st.lists(_any_of(ARRAY_ARRAY), max_size=2), + _any_of(ARRAY_BYTES), + st.lists(_any_of(BYTES_BYTES), max_size=2), + ).map(lambda parts: (*parts[0], parts[1], *parts[2])) + + +@st.composite +def rank_matched_shards(draw: st.DrawFn) -> tuple[object, ...]: + """Chains whose shard has the document's rank, with arbitrary extents. + + `codec_chains` draws a shard's `chunk_shape` freely, so it almost never + has the right rank and the rank check short-circuits before geometry is + reached. Matching the rank is what exposes divisibility to the fuzzer. + """ + extents = tuple(draw(st.lists(st.integers(min_value=1, max_value=48), min_size=2, max_size=2))) + return ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": extents, + "codecs": (_LITTLE,), + "index_codecs": (_LITTLE,), + }, + }, + ) + + +def document(codecs: object, **overrides: object) -> dict[str, object]: + """A v3 array document around `codecs`, valid apart from what is passed.""" + return { + "zarr_format": 3, + "node_type": "array", + "shape": (64, 64), + "data_type": "uint16", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (32, 32)}}, + "chunk_key_encoding": "default", + "codecs": codecs, + **overrides, + } + + +def _divisors(value: int) -> list[int]: + return [candidate for candidate in range(1, value + 1) if value % candidate == 0] + + +@st.composite +def valid_documents(draw: st.DrawFn) -> dict[str, object]: + """Documents that must validate clean, correlated by construction. + + A transpose permutes the grid and the shard's inner shape follows it, + so the inner extents are checked against the axis they actually meet — + including a rectilinear axis whose chunks differ, where only a common + divisor of every length will do. + """ + rank = draw(st.integers(min_value=1, max_value=3)) + rectilinear = draw(st.booleans()) + # Per axis: the lengths its chunks take, and an extent dividing all of them. + lengths: list[tuple[int, ...]] = [] + for _ in range(rank): + if rectilinear: + axis = tuple(draw(st.lists(st.sampled_from([8, 12, 16, 24]), min_size=1, max_size=3))) + else: + axis = (draw(st.sampled_from([8, 12, 16, 24])),) + lengths.append(axis) + inner = [draw(st.sampled_from(_divisors(gcd(*axis, axis[0])))) for axis in lengths] + order = tuple(draw(st.permutations(range(rank)))) + + if rectilinear: + grid: dict[str, object] = { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": tuple(lengths)}, + } + shape = tuple(sum(axis) for axis in lengths) + else: + grid = { + "name": "regular", + "configuration": {"chunk_shape": tuple(axis[0] for axis in lengths)}, + } + shape = tuple(axis[0] * 2 for axis in lengths) + + shard = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": tuple(inner[axis] for axis in order), + "codecs": (_LITTLE,), + "index_codecs": (_LITTLE,), + }, + } + return { + "zarr_format": 3, + "node_type": "array", + "shape": shape, + "data_type": "uint16", + "fill_value": 0, + "chunk_grid": grid, + "chunk_key_encoding": "default", + "codecs": ({"name": "transpose", "configuration": {"order": order}}, shard), + } + + +__all__ = [ + "ARRAY_ARRAY", + "ARRAY_BYTES", + "BYTES_BYTES", + "JSON_VALUES", + "codec_chains", + "document", + "rank_matched_shards", + "valid_documents", +] diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py new file mode 100644 index 0000000000..9b73a077b5 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -0,0 +1,150 @@ +"""Property tests over codec chains drawn from the codec types themselves. + +The existing totality test feeds arbitrary JSON to the document +validators, which is the right guard for the structural layer and no guard +at all for this one: a random object never names a codec, so it never +dispatches an entity rule. `test_the_chain_strategy_reaches_the_rules` +exists so that stays visible — a refactor that silently stops dispatching +fails here instead of staying green. +""" + +from __future__ import annotations + +import contextlib +import pkgutil +from typing import TYPE_CHECKING + +import pytest +from hypothesis import HealthCheck, given, settings + +import zarr_metadata.v3.codec +from tests.rules.strategies import ( + ARRAY_ARRAY, + ARRAY_BYTES, + BYTES_BYTES, + codec_chains, + document, + rank_matched_shards, + valid_documents, +) +from zarr_metadata.rules import parse_array_metadata_v3, validate_array_metadata_v3 +from zarr_metadata.v3.codec.kind import codec_kind_of_name + +if TYPE_CHECKING: + from collections.abc import Mapping + +_SLOW = settings(max_examples=300, deadline=None, suppress_health_check=list(HealthCheck)) + + +def test_the_strategies_cover_every_codec_the_package_models() -> None: + # The kind tuples are hand-written; a new codec module must join one. + drawn = { + entry.__annotations__["name"].__args__[0] + for entry in (*ARRAY_ARRAY, *ARRAY_BYTES, *BYTES_BYTES) + } + modelled = { + value + for info in pkgutil.iter_modules(zarr_metadata.v3.codec.__path__) + if not info.name.startswith("_") + for attribute, value in vars( + __import__(f"zarr_metadata.v3.codec.{info.name}", fromlist=["_"]) + ).items() + if attribute.endswith("_CODEC_NAME") and isinstance(value, str) + } + assert modelled == drawn + for kinds, expected in ((ARRAY_ARRAY, "array_array"), (ARRAY_BYTES, "array_bytes")): + for entry in kinds: + assert codec_kind_of_name(entry.__annotations__["name"].__args__[0]) == expected + + +@given(codec_chains()) +@_SLOW +def test_a_well_ordered_chain_always_produces_a_verdict(codecs: tuple[object, ...]) -> None: + """Untrusted input gets judged, never raises, however odd the values.""" + assert isinstance(validate_array_metadata_v3(document(codecs)), tuple) + + +# The rules a chain of arbitrary codec configurations reliably exercises, +# with the observed rate over 600 examples. The three chain rules absent +# here are out of this strategy's reach by construction, not by accident: +# pipeline ordering cannot fire because the chain is assembled in order, +# and variable-length data types and variable-size index codecs need a +# correlated data type and index chain that only a hand-written case +# supplies. Those have their own tests in `test_v3_array_rules.py`. +_WITNESSES: Mapping[str, str] = { + "rank against the incoming array": "incoming array has", # 63% + "positive chunk extents": "expected a positive chunk extent", # 34% + "transpose order is a permutation": "expected a permutation", # 21% + "endianness for multi-byte types": "endian is required", # 20% +} + + +def test_the_chain_strategy_reaches_the_rules() -> None: + reached: set[str] = set() + + @given(codec_chains()) + @_SLOW + def sample(codecs: tuple[object, ...]) -> None: + messages = [problem.message for problem in validate_array_metadata_v3(document(codecs))] + reached.update( + label + for label, witness in _WITNESSES.items() + if any(witness in message for message in messages) + ) + + sample() + assert reached == set(_WITNESSES) + + +def test_the_shard_strategy_reaches_chunk_geometry() -> None: + # A shard whose rank matches is what exposes divisibility: drawn freely + # it almost never has the right rank, and the rank check returns first. + reached = [False] + + @given(rank_matched_shards()) + @_SLOW + def sample(codecs: tuple[object, ...]) -> None: + if any( + "does not evenly divide" in problem.message + for problem in validate_array_metadata_v3(document(codecs)) + ): + reached[0] = True + + sample() + assert reached[0] + + +@given(valid_documents()) +@_SLOW +def test_documents_valid_by_construction_are_accepted(doc: Mapping[str, object]) -> None: + """The accept side: extents drawn, then inner shapes from their divisors. + + Every fix to this layer has made it stricter, and almost nothing drawn + from the codec types is valid, so this is the only property guarding + the other direction over generated input. + """ + problems = validate_array_metadata_v3(doc) + assert problems == (), [problem.message for problem in problems] + + +@given(valid_documents()) +@_SLOW +def test_the_parser_agrees_with_the_validator_on_valid_documents( + doc: Mapping[str, object], +) -> None: + assert parse_array_metadata_v3(doc) is not None + + +@pytest.mark.parametrize( + "strategy", [codec_chains(), rank_matched_shards()], ids=["chains", "shards"] +) +def test_no_chain_makes_the_parser_raise_anything_but_its_own_error(strategy: object) -> None: + from zarr_metadata.model import MetadataValidationError + + @given(strategy) # type: ignore[arg-type] + @_SLOW + def sample(codecs: tuple[object, ...]) -> None: + with contextlib.suppress(MetadataValidationError): + parse_array_metadata_v3(document(codecs)) + + sample() From a6c84eaa1d642f25357fceb639fbf997dd711f9c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 14:08:11 +0200 Subject: [PATCH 019/107] fix(zarr-metadata): judge a shard behind an unmodelled codec, and check tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Differential testing against the pre-ArrayParts tree found 390 documents the old validator rejected and the new one accepted. All of them are one regression: gating the shard's inner and index pipelines on `incoming`. Both are determined by the sharding codec's own `chunk_shape` and by the spec — the index is a `uint64` array whatever reaches the codec — so neither should ever have waited on upstream. It was a consequence of making `data_type` non-optional: with nothing to pair the grid with, both starts collapsed. The reasoning behind that change held for documents and failed inside a shard, where the grid is known and the element type is not, so `data_type` is optional again while `ArrayParts | None` keeps its own meaning of "no array here". Also: an unusable `data_type` costs itself and no longer hides the geometry, and `ChunkGrid.permuted` declines on a non-permutation instead of raising `IndexError` one careless caller away from a validator. Tests are now type-checked. They were covered by nothing — pyright was configured for `src` alone and the repo's mypy hook resolves this package's imports as `Any` — which is how a `TYPE_CHECKING` import of a type deleted three commits ago survived. It also showed that `from_key_value(to_key_value())`, the round trip the models advertise, did not type-check: `Mapping` is invariant in its key type, so a mapping keyed by literal store keys is not a `Mapping[str, bytes]`. Widened. A mutation audit put the suite's kill rate at 67%, with the largest hole in the `reads` gate: `st.from_type` honours the TypedDicts, so no strategy produced an ill-typed configuration member, and four one-token changes to the gate made `validate_*` raise while the suite stayed green. `corrupted_chains` covers it; the three mutations were verified to fail now. Two redundant property tests are gone, and the reach claim in the strategies module is replaced with measured figures — the previous 39% to 94% and its short-circuit explanation did not reproduce. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.bugfix.md | 8 +++ packages/zarr-metadata/changes/4379.misc.md | 22 +++++++- packages/zarr-metadata/justfile | 4 +- packages/zarr-metadata/pyproject.toml | 23 +++++++- .../src/zarr_metadata/model/_array.py | 13 +---- .../src/zarr_metadata/model/_group.py | 20 ++----- .../src/zarr_metadata/rules/_chunk_grid.py | 7 ++- .../rules/_entities/bytes_codec.py | 2 +- .../zarr_metadata/rules/_entities/sharding.py | 19 +++++-- .../src/zarr_metadata/rules/_registry.py | 10 ++-- .../src/zarr_metadata/rules/_spec.py | 19 ++++--- .../zarr-metadata/tests/model/test_array.py | 10 ++-- .../zarr-metadata/tests/model/test_group.py | 9 ++- .../tests/model/test_pydantic.py | 7 ++- .../zarr-metadata/tests/rules/strategies.py | 55 ++++++++++++++++--- .../tests/rules/test_chain_properties.py | 49 +++++++++++------ .../tests/rules/test_chunk_grid.py | 52 ++++++++++++++++++ .../tests/rules/test_registry.py | 24 ++++++-- .../tests/rules/test_v3_array_rules.py | 4 +- 19 files changed, 267 insertions(+), 90 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md index 64d811f733..42dbed80a6 100644 --- a/packages/zarr-metadata/changes/4379.bugfix.md +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -47,3 +47,11 @@ grid uniform on one axis only is judged on that axis and declines on the other; and a shard index is judged against its own derived shape — chunks-per-shard plus a trailing dimension of 2 — rather than against no shape at all. + +An unmodelled codec in front of a sharding codec — every `numcodecs.*` +filter is one — silenced the whole shard interior: its inner pipeline and +its index pipeline went unjudged. Both are determined by the sharding +codec's own `chunk_shape` and by the spec (the index is `uint64`), so +neither waits on what reached the codec. Likewise an unusable `data_type` +no longer hides the geometry, and a chunk grid declines to permute by +anything that is not a permutation rather than raising `IndexError`. diff --git a/packages/zarr-metadata/changes/4379.misc.md b/packages/zarr-metadata/changes/4379.misc.md index 50a6691f84..429e163c3c 100644 --- a/packages/zarr-metadata/changes/4379.misc.md +++ b/packages/zarr-metadata/changes/4379.misc.md @@ -7,9 +7,11 @@ The pre-existing totality test feeds arbitrary JSON to the document validators; a random object never names a codec, so it dispatched no entity rule at all. Drawing codecs from their own types and assembling the chain by pipeline kind (`array->array`* `array->bytes` `bytes->bytes`*) -takes the proportion of documents reaching a chain rule from 0% to 94% — -ordering matters because a chain in the wrong order is rejected before any -other rule runs. +dispatches an entity rule for every document, where arbitrary JSON +dispatched none; 86% report a composition problem, against 78% for a flat +list of the same codecs. Ordering earns that by guaranteeing an +`array->bytes` codec is present, not by avoiding an early exit — nothing +short-circuits on a misordered chain. Two properties are now covered that were not: that a chain of arbitrary configurations always produces a verdict rather than raising, and that @@ -21,3 +23,17 @@ against, since fewer than 5% of generated documents are valid. The reach of each strategy is asserted rather than assumed, so a change that silently stops dispatching fails loudly instead of leaving a green property run that tests nothing. + +`st.from_type` honours the TypedDicts exactly, so it cannot produce a +configuration member of the wrong *type* — which is precisely what the +per-member `reads` gate exists to stand rules down for. A corrupting +strategy covers that: four separate one-token changes to the gate make +`validate_*` raise `TypeError` on ordinary malformed metadata, and none of +them was detectable before. + +The package's tests are now type-checked along with its sources. They were +checked by nothing: pyright was configured for `src` alone, and the repo's +mypy hook resolves this package's imports as `Any`. Turning it on found a +`TYPE_CHECKING` import of a type deleted three commits earlier, and that +`from_key_value(to_key_value())` — the round trip the models advertise — +did not type-check, because `Mapping` is invariant in its key type. diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index 0f1861ed7d..d9a0d2a5f6 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -19,9 +19,9 @@ pyright_version := "1.1.404" # CI runs pyright on python 3.11; the pinned pyright predates 3.14, whose # stdlib it cannot parse, so pin the interpreter to match CI. -# Type-check the package sources +# Type-check the package, sources and tests alike typecheck: - uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright src + uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright # Run everything CI runs for this package check: lint typecheck test docs-check diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 814eeccef4..bfe7dce924 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -129,11 +129,32 @@ checks = [ # class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel # relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] -include = ["src"] +include = ["src", "tests"] +# `tests` is a package imported as `tests.*`, which pytest resolves from the +# rootdir; pyright needs the same root on its search path. +extraPaths = ["."] enableExperimentalFeatures = true typeCheckingMode = "strict" pythonVersion = "3.11" +# Tests are checked, but not for the inference strictness the sources are held +# to: test bodies are full of untyped JSON literals whose inferred types are of +# no interest, and strict mode reports about ninety of those. What is kept is +# everything a rename or a signature change breaks -- a dangling import, a call +# that no longer type-checks, an argument of the wrong type -- which is the +# reason for checking tests at all. Tests also reach into private modules and +# define functions purely to be refused at registration, both deliberately. +[[tool.pyright.executionEnvironments]] +root = "tests" +reportMissingTypeArgument = false +reportPrivateUsage = false +reportUnknownArgumentType = false +reportUnknownLambdaType = false +reportUnknownMemberType = false +reportUnknownParameterType = false +reportUnknownVariableType = false +reportUnusedFunction = false + [tool.towncrier] # Fragments for this package live alongside the package source, separate # from the parent zarr-python `changes/` directory, so a PR touching only diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 26a69a304a..520c90ebbd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -31,16 +31,13 @@ from zarr_metadata.v2.array import ( ZarrV2ArrayDimensionSeparator, ZarrV2ArrayMetadataJSON, - ZarrV2ArrayMetadataStoreKey, ZarrV2ArrayOrder, ZarrV2DataTypeMetadata, ) - from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3.array import ( ZarrV3ArrayMetadataJSON, - ZarrV3ArrayMetadataStoreKey, ZarrV3ExtensionField, ) @@ -325,9 +322,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY)) - def to_key_value( - self, *, indent: int | str | None = None - ) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]: + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)} @@ -488,14 +483,12 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata: return cls.from_json({**zarray, "attributes": zattrs}) return cls.from_json(zarray) - def to_key_value( - self, *, indent: int | str | None = None - ) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zarray` # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} - out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + out: dict[str, bytes] = { ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent) } if self.attributes is not UNSET: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 5519e6fbb9..12e8324871 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -35,12 +35,10 @@ if TYPE_CHECKING: from zarr_metadata._common import JSONValue - from zarr_metadata.v2.attributes import ZarrV2AttributesStoreKey - from zarr_metadata.v2.consolidated import ZarrV2ConsolidatedMetadataStoreKey - from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON, ZarrV2GroupMetadataStoreKey + from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON from zarr_metadata.v3.array import ZarrV3ExtensionField from zarr_metadata.v3.consolidated import ZarrV3ConsolidatedMetadataJSON - from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON, ZarrV3GroupMetadataStoreKey + from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON class ZarrV3GroupMetadataPartial(TypedDict, total=False): @@ -179,9 +177,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY)) - def to_key_value( - self, *, indent: int | str | None = None - ) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]: + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)} @@ -334,14 +330,12 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata: return cls.from_json({**zgroup, "attributes": zattrs}) return cls.from_json(zgroup) - def to_key_value( - self, *, indent: int | str | None = None - ) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]: + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: # Attributes live only in the sibling `.zattrs` file; the `.zgroup` # document must exclude them. The `.zattrs` key is present exactly # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} - out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = { + out: dict[str, bytes] = { ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent) } if self.attributes is not UNSET: @@ -429,9 +423,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata: return cls.from_json(load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY)) - def to_key_value( - self, *, indent: int | str | None = None - ) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]: + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent) } diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py index 0b0ab3148a..1ac924ec6a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py @@ -172,8 +172,13 @@ def permuted(self, order: Sequence[int]) -> ChunkGrid: A transposed grid is still a grid — permuting a regular one gives a regular one — but it is no longer the grid the document wrote, so the metadata does not survive the trip. + + Declines on anything that is not a permutation of this grid's rank. + The caller checks that too and reports it, but an order is only + shape-validated as a tuple of integers, so this must not be the + thing that decides whether a validator raises `IndexError`. """ - if self.extents is None or len(order) != len(self.extents): + if self.extents is None or sorted(order) != list(range(len(self.extents))): return ChunkGrid(self.rank, None) return ChunkGrid.derived(tuple(self.extents[axis] for axis in order)) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py index 0c06b5e468..87dc3601ee 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py @@ -23,7 +23,7 @@ def data_type_has_a_raw_byte_representation( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None ) -> tuple[ValidationProblem, ...]: - if incoming is None: + if incoming is None or incoming.data_type is None: return () shape_verdict = validate_known_entity_metadata(DATA_TYPE, incoming.data_type) if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py index 14c91720dd..86b8fc64d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, cast from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._chunk_grid import ChunkGrid, shard_index_grid +from zarr_metadata.rules._chunk_grid import UNKNOWN_GRID, ChunkGrid, shard_index_grid from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems from zarr_metadata.rules._registry import entity_rule, run_chain_rules from zarr_metadata.rules._spec import ArrayParts @@ -116,10 +116,10 @@ def inner_pipelines_are_pipelines( against the inner chunk, and its own transitions carry on from there. The `index_codecs` chain encodes the shard index: a `uint64` array of chunks-per-shard plus a trailing dimension of 2, derived by - `zarr_metadata.rules._chunk_grid.shard_index_shape`. + `zarr_metadata.rules._chunk_grid.shard_index_grid`. """ inner = configuration["chunk_shape"] - if not isinstance(inner, tuple) or incoming is None: + if not isinstance(inner, tuple): inner_start: ArrayParts | None = None index_start: ArrayParts | None = None else: @@ -128,8 +128,17 @@ def inner_pipelines_are_pipelines( # the chunk this codec receives, so the inner pipeline is built # exactly like the document's own, and the index's grid follows # from the two together. - inner_start = incoming.with_grid(ChunkGrid.regular(extents)) - index_start = ArrayParts(shard_index_grid(incoming.grid, extents), "uint64") + # + # Both come from this codec's own configuration and from the spec, + # so neither waits on what reached the codec. An unreadable codec + # upstream costs the element type and the enclosing extents; it + # does not make the inner chunk shape unknown, and the index is a + # `uint64` array whatever happens before it. + outer = incoming.grid if incoming is not None else UNKNOWN_GRID + inner_start = ArrayParts( + ChunkGrid.regular(extents), incoming.data_type if incoming is not None else None + ) + index_start = ArrayParts(shard_index_grid(outer, extents), "uint64") problems: list[ValidationProblem] = [] for key in ("codecs", "index_codecs"): entries = configuration[key] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py index 104a1dc3db..88c9b76fcb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py @@ -332,13 +332,15 @@ def chain_initial_spec(document: Mapping[str, object]) -> ArrayParts | None: """What enters a document's top-level codec chain. The parts a chunk pipeline encodes are the chunks of the document's - chunk grid. A document whose `data_type` is not a metadata field has - already been rejected structurally, so there is nothing to describe. + chunk grid. A `data_type` that is not a metadata field has been + rejected structurally already, but it costs only itself: the grid is + still readable, and the geometry rules should still report what they + can rather than making the reader fix one fault to discover the rest. """ data_type = document.get("data_type") - if entity_name(data_type) is None: - return None grid = ChunkGrid.of(document.get("chunk_grid"), document.get("shape")) + if entity_name(data_type) is None: + return ArrayParts(grid, None) return ArrayParts(grid, cast("ZarrV3MetadataFieldJSON", data_type)) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py index 9127ae89a5..72ac5711a9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py @@ -43,20 +43,23 @@ class ArrayParts: divide *every* chunk, which under a rectilinear grid is several different lengths. - `data_type` is never absent. The only documents that cannot supply one - are documents the structural layer has already rejected, so "a real - array whose type we do not know" is not a state worth modelling; when - nothing is known, there are no `ArrayParts` at all. It is the - metadata-field value verbatim, because rules compare it by name. + `data_type` is the metadata-field value verbatim, because rules compare + it by name, and it is `None` where the element type is undetermined + while the array itself is not. That happens inside a shard: the inner + grid is the sharding codec's own `chunk_shape` whatever reached it, so + an unreadable codec upstream costs the type and not the parts. `None` + in place of the whole value means something else again — that there is + no array here at all, past the array->bytes boundary or beyond a codec + that could have changed anything. """ grid: ChunkGrid - data_type: ZarrV3MetadataFieldJSON + data_type: ZarrV3MetadataFieldJSON | None def with_grid(self, grid: ChunkGrid) -> ArrayParts: return replace(self, grid=grid) - def with_data_type(self, data_type: ZarrV3MetadataFieldJSON) -> ArrayParts: + def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArrayParts: return replace(self, data_type=data_type) @@ -73,7 +76,7 @@ def with_data_type(self, data_type: ZarrV3MetadataFieldJSON) -> ArrayParts: def spec_transition(codec: str) -> Callable[[SpecTransition], SpecTransition]: - """Register how `codec` transforms an incoming `ArraySpec`. + """Register how `codec` transforms the `ArrayParts` it receives. Only array->array codecs need one: array->bytes and bytes->bytes codecs end shape propagation by construction, so registering a diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 408308de59..ab68f9ad23 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -5,7 +5,7 @@ import json from collections import UserDict from collections.abc import Callable -from typing import TYPE_CHECKING, get_args +from typing import TYPE_CHECKING, cast, get_args import pytest from typing_extensions import Unpack @@ -40,6 +40,8 @@ from zarr_metadata.model._validation import _prefix, arrays_to_tuples if TYPE_CHECKING: + from collections.abc import Mapping + from zarr_metadata._common import JSONValue from zarr_metadata.v2 import ZarrV2CodecMetadata @@ -977,7 +979,7 @@ def test_parse_metadata_field_materializes_abstract_containers() -> None: assert isinstance(parsed, dict) assert parsed == {"name": "example", "configuration": {"values": (0, 1)}} - assert type(parsed["configuration"]) is dict + assert type(cast("Mapping[str, object]", parsed)["configuration"]) is dict def test_metadata_field_type_guard_rejects_abstract_mapping() -> None: @@ -1572,7 +1574,7 @@ def test_configuration_values_must_be_json() -> None: def test_v3_extension_keys_must_be_strings() -> None: """A non-string top-level key cannot be represented by a v3 document type.""" - doc: dict[object, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) + doc: dict[object, object] = {**ZarrV3ArrayMetadata.create_default().to_json()} doc[1] = {"must_understand": False} assert [(problem.loc, problem.kind) for problem in validate_array_metadata_v3(doc)] == [ ((), "invalid_type") @@ -1771,7 +1773,7 @@ def test_v2_absent_dimension_separator_means_dot() -> None: del doc["dimension_separator"] model = ZarrV2ArrayMetadata.from_json(doc) assert model.dimension_separator == "." - assert model.to_json()["dimension_separator"] == "." + assert cast("Mapping[str, object]", model.to_json())["dimension_separator"] == "." def test_v2_from_key_value_without_separator_means_dot() -> None: diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index b0e451b0a0..a526731f00 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -5,6 +5,7 @@ import json from collections import UserDict from collections.abc import Callable +from typing import TYPE_CHECKING, cast import pytest @@ -33,6 +34,10 @@ # --- ZarrV3GroupMetadata --------------------------------------------------- +if TYPE_CHECKING: + from collections.abc import Mapping + + def test_group_v3_roundtrip() -> None: """A v3 group document round-trips through the model unchanged.""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": (1, 2)}} @@ -163,7 +168,9 @@ def test_group_guards_reject_noncanonical_nested_json() -> None: assert not is_group_metadata_v3(v3) assert not is_group_metadata_v2(v2) assert parse_group_metadata_v3(v3)["extension"] == (0, 1) - assert parse_group_metadata_v2(v2)["attributes"] == {"values": (0, 1)} + assert cast("Mapping[str, object]", parse_group_metadata_v2(v2))["attributes"] == { + "values": (0, 1) + } def test_group_v3_extension_fields_are_validated() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic.py b/packages/zarr-metadata/tests/model/test_pydantic.py index e5714bb8fb..90502803b3 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic.py +++ b/packages/zarr-metadata/tests/model/test_pydantic.py @@ -24,8 +24,8 @@ messages intact. """ -from collections.abc import Mapping -from typing import Annotated, Generic, TypeVar +from collections.abc import Mapping, Sequence +from typing import Annotated, Generic, TypeVar, cast import pytest from pydantic import ( @@ -234,8 +234,9 @@ def _canonicalize(cls, data: object) -> object: if isinstance(doc[key], str): doc[key] = {"name": doc[key]} for key in ("codecs", "storage_transformers"): + entries = cast("Sequence[object]", doc.get(key, ())) doc[key] = tuple( - {"name": item} if isinstance(item, str) else item for item in doc.get(key, ()) + {"name": item} if isinstance(item, str) else item for item in entries ) doc.setdefault("attributes", {}) return doc diff --git a/packages/zarr-metadata/tests/rules/strategies.py b/packages/zarr-metadata/tests/rules/strategies.py index f3de60a893..1884af8e31 100644 --- a/packages/zarr-metadata/tests/rules/strategies.py +++ b/packages/zarr-metadata/tests/rules/strategies.py @@ -5,12 +5,17 @@ `codec_chains` draws each codec with `st.from_type` over its own `TypedDict`, so the values are structurally plausible and almost always -semantically wrong. That is what the rejection side wants. Crucially the -chain is assembled by *kind* — `array->array`* `array->bytes` -`bytes->bytes`* — because a chain in the wrong order is rejected by the -ordering rule before any other rule runs, and a flat list of random codecs -is in the wrong order most of the time. Ordering the chain takes the -proportion of documents that reach a chain rule from 39% to 94%. +semantically wrong. That is what the rejection side wants. The chain is +assembled by *kind* — `array->array`* `array->bytes` `bytes->bytes`* — +which guarantees an `array->bytes` codec is present: measured over 3000 +documents, an entity rule is dispatched for 100% of ordered chains and one +reports a problem for 86%, against 78% for a flat list of the same codecs. +Nothing short-circuits on a misordered chain, so the gain is coverage of +the array->bytes codecs rather than avoided early exit. + +What this cannot reach, because `st.from_type` honours the TypedDicts +exactly, is a configuration member of the wrong *type* — +`corrupted_chains` exists for that. `valid_documents` builds documents that must be accepted, by construction: extents are drawn, then inner chunk shapes are drawn from their divisors. @@ -27,7 +32,7 @@ from __future__ import annotations from math import gcd -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from hypothesis import strategies as st @@ -52,10 +57,13 @@ ), max_leaves=5, ) -st.register_type_strategy(JSONValue, JSON_VALUES) +# `JSONValue` is a `TypeAliasType`, which `register_type_strategy` does not +# accept in its signature but does resolve at runtime — it is exactly the +# forward reference `from_type` fails on. +st.register_type_strategy(JSONValue, JSON_VALUES) # type: ignore[arg-type] # The codec TypedDicts, by pipeline kind. Hand-written because there is no -# name-to-type table to derive it from; `test_strategies.py` asserts it +# name-to-type table to derive it from; `test_chain_properties.py` asserts it # covers every codec the package models. ARRAY_ARRAY = (TransposeCodecObject, CastValueCodecObject, ScaleOffsetCodecObject) ARRAY_BYTES = (BytesCodecObject, ShardingIndexedCodecObject) @@ -173,12 +181,41 @@ def valid_documents(draw: st.DrawFn) -> dict[str, object]: } +@st.composite +def corrupted_chains(draw: st.DrawFn) -> tuple[object, ...]: + """A well-typed chain with one configuration member replaced by any JSON. + + `st.from_type` honours the TypedDicts, so it never produces an ill-typed + member — and an ill-typed member is exactly what the `reads` gate exists + to stand rules down for. Without this, the gate that makes + `configuration["level"]` safe inside a rule has no generated coverage, + and a validator that raises `TypeError` on ordinary malformed metadata + looks identical to one that does not. + """ + chain = list(draw(codec_chains())) + index = draw(st.integers(min_value=0, max_value=len(chain) - 1)) + codec = chain[index] + if not isinstance(codec, dict): + return tuple(chain) + entry = cast("dict[str, object]", dict(codec)) + configuration = entry.get("configuration") + if not isinstance(configuration, dict) or len(cast("dict[str, object]", configuration)) == 0: + return tuple(chain) + members = cast("dict[str, object]", dict(configuration)) + member = draw(st.sampled_from(sorted(members))) + members[member] = draw(JSON_VALUES) + entry["configuration"] = members + chain[index] = entry + return tuple(chain) + + __all__ = [ "ARRAY_ARRAY", "ARRAY_BYTES", "BYTES_BYTES", "JSON_VALUES", "codec_chains", + "corrupted_chains", "document", "rank_matched_shards", "valid_documents", diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py index 9b73a077b5..d50dd6482c 100644 --- a/packages/zarr-metadata/tests/rules/test_chain_properties.py +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -10,11 +10,9 @@ from __future__ import annotations -import contextlib import pkgutil from typing import TYPE_CHECKING -import pytest from hypothesis import HealthCheck, given, settings import zarr_metadata.v3.codec @@ -23,17 +21,25 @@ ARRAY_BYTES, BYTES_BYTES, codec_chains, + corrupted_chains, document, rank_matched_shards, valid_documents, ) -from zarr_metadata.rules import parse_array_metadata_v3, validate_array_metadata_v3 +from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3.codec.kind import codec_kind_of_name if TYPE_CHECKING: from collections.abc import Mapping -_SLOW = settings(max_examples=300, deadline=None, suppress_health_check=list(HealthCheck)) +# Drawing a codec chain and validating a document is slow per example and +# the outer functions below run a whole @given loop of their own, so the two +# timing checks are suppressed; nothing else is. +_SLOW = settings( + max_examples=300, + deadline=None, + suppress_health_check=(HealthCheck.too_slow, HealthCheck.data_too_large), +) def test_the_strategies_cover_every_codec_the_package_models() -> None: @@ -127,24 +133,35 @@ def test_documents_valid_by_construction_are_accepted(doc: Mapping[str, object]) assert problems == (), [problem.message for problem in problems] -@given(valid_documents()) +@given(corrupted_chains()) @_SLOW -def test_the_parser_agrees_with_the_validator_on_valid_documents( - doc: Mapping[str, object], +def test_an_ill_typed_configuration_member_is_reported_not_raised( + codecs: tuple[object, ...], ) -> None: - assert parse_array_metadata_v3(doc) is not None + """A member of the wrong type stands its rules down; it never crashes one. + + This is the property the `reads` gate exists for, and the only generated + input that exercises it: `st.from_type` honours the TypedDicts, so every + other strategy here produces well-typed members. Without this, four + separate one-token changes to the gate leave the suite green while + `validate_*` raises `TypeError` on ordinary malformed metadata. + """ + problems = validate_array_metadata_v3(document(codecs)) + assert isinstance(problems, tuple) -@pytest.mark.parametrize( - "strategy", [codec_chains(), rank_matched_shards()], ids=["chains", "shards"] -) -def test_no_chain_makes_the_parser_raise_anything_but_its_own_error(strategy: object) -> None: - from zarr_metadata.model import MetadataValidationError +def test_the_corrupting_strategy_reaches_the_gate() -> None: + # It has to produce ill-typed members, or the property above is vacuous. + reached = [False] - @given(strategy) # type: ignore[arg-type] + @given(corrupted_chains()) @_SLOW def sample(codecs: tuple[object, ...]) -> None: - with contextlib.suppress(MetadataValidationError): - parse_array_metadata_v3(document(codecs)) + if any( + problem.kind == "invalid_type" + for problem in validate_array_metadata_v3(document(codecs)) + ): + reached[0] = True sample() + assert reached[0] diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index 3321300552..fe91d7ffc8 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -214,3 +214,55 @@ def test_error_a_bad_inner_extent_costs_that_axis_and_nothing_else() -> None: assert any("positive chunk extent" in message for message in messages) assert any("order has 3 entries" in message for message in messages) assert any("endian is required" in message for message in messages) + + +# An unmodelled codec is the ordinary case, not an exotic one: every +# `numcodecs.*` filter is one. What it costs must be only what it actually +# obscures. +_UNMODELLED = { + "name": "numcodecs.delta", + "configuration": {"dtype": " None: + # The inner grid is this codec's own `chunk_shape` and the index is a + # uint64 array, whatever reached the codec. Neither waits on upstream. + nested = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (16, 16), + "codecs": ({"name": "transpose", "configuration": {"order": (0, 1, 2)}}, "bytes"), + "index_codecs": ("bytes",), + }, + } + messages = [ + problem.message + for problem in validate_array_metadata_v3( + {**BASE, "data_type": "uint16", "chunk_grid": REGULAR, "codecs": (_UNMODELLED, nested)} + ) + ] + assert any("order has 3 entries" in message for message in messages) + assert any("uint64" in message for message in messages) + + +def test_an_unusable_data_type_does_not_hide_the_geometry() -> None: + # `data_type` costs itself and nothing else: the grid is still readable, + # so a reader sees every fault at once instead of one per round trip. + messages = [ + problem.message + for problem in validate_array_metadata_v3( + {**BASE, "data_type": 5, "chunk_grid": REGULAR, "codecs": (_shard((7, 7)),)} + ) + ] + assert any("expected a metadata field" in message for message in messages) + assert sum("does not evenly divide" in message for message in messages) == 2 + + +def test_permuting_by_a_non_permutation_declines_rather_than_raising() -> None: + # `order` is shape-validated only as a tuple of integers, so this must + # not be what decides whether a validator raises IndexError. + grid = ChunkGrid.regular((4, 4)) + for order in ((5, 0), (0, 0), (-1, 0)): + assert grid.permuted(order) == ChunkGrid(2, None) diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py index 0970fa620f..99450f1c47 100644 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -37,7 +37,7 @@ from collections.abc import Mapping from zarr_metadata.model._validation import ValidationProblem - from zarr_metadata.rules._spec import ArraySpec + from zarr_metadata.rules._spec import ArrayParts from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME @@ -137,7 +137,11 @@ def test_error_entity_rule_requiring_an_unknown_key() -> None: with pytest.raises(ValueError, match="could never fire"): @entity_rule(ZARR_V3_ARRAY, CHUNK_GRID, "regular", requires=frozenset({"shapee"})) - def _misspelled(configuration: object, document: object) -> tuple[()]: # pragma: no cover + def _misspelled( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, + ) -> tuple[()]: # pragma: no cover - refused at registration return () @@ -147,7 +151,11 @@ def test_error_entity_rule_for_an_unmodelled_entity() -> None: with pytest.raises(ValueError, match="no shape validator"): @entity_rule(ZARR_V3_ARRAY, CHUNK_GRID, "hilbert") - def _unmodelled(configuration: object, document: object) -> tuple[()]: # pragma: no cover + def _unmodelled( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, + ) -> tuple[()]: # pragma: no cover - refused at registration return () @@ -158,7 +166,11 @@ def test_error_entity_rule_for_name_modelled_only_at_another_extension_point() - with pytest.raises(ValueError, match="no shape validator"): @entity_rule(ZARR_V3_ARRAY, CODECS, "regular") - def _wrong_extension_point(configuration: object, document: object) -> tuple[()]: + def _wrong_extension_point( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, + ) -> tuple[()]: # pragma: no cover - refused at registration return () @@ -187,7 +199,7 @@ def test_error_entity_rule_reads_an_unmodelled_member() -> None: def _unmodelled_member( configuration: Mapping[str, object], document: Mapping[str, object], - incoming: ArraySpec, + incoming: ArrayParts | None, ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered return () @@ -201,6 +213,6 @@ def test_error_entity_rule_reads_an_optional_member() -> None: def _subscripts_an_optional_member( configuration: Mapping[str, object], document: Mapping[str, object], - incoming: ArraySpec, + incoming: ArrayParts | None, ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered return () diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 36d0909056..2cc1455de1 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast import pytest @@ -552,7 +552,7 @@ def test_unknown_member_survives_a_round_trip() -> None: } model = ZarrV3ArrayMetadata.from_json(json.loads(json.dumps(raw))) emitted = model.to_json() - codec = emitted["codecs"][0] + codec = cast("Mapping[str, Any]", emitted["codecs"][0]) assert codec["configuration"]["numThreads"] == 4 From 13d5018b6c41588fc81772856722b865379583d4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 14:16:06 +0200 Subject: [PATCH 020/107] feat(zarr-metadata): judge blosc values, and its conditional typesize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `typesize` is the only conditionally-required configuration member of any entity this package models: "Positive integer specifying the stride in bytes over which shuffling is performed. Required unless `shuffle` is `"noshuffle"`, in which case the value is ignored." A TypedDict cannot express that, so `BloscCodecConfiguration` declares it `NotRequired` and nothing supplied the condition — leaving the member the spec singles out as required as the one member of a blosc configuration that could always be omitted, while `blocksize`, which the spec never marks required, was. A rule supplies it, alongside the value constraints the shape validator cannot state: `clevel` in [0, 9], `typesize` positive, `blocksize` non-negative. blosc leaves the deliberately-rule-free list. These rules were written for the stacked #4380, which is based on this PR's pre-redesign head and would not apply to it; they are brought down here because this is the PR that makes `typesize` optional, and #4380 should drop its blosc half when it rebases. Verified against the spec's own example document and against what zarr-python writes for blosc with and without shuffling. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.7.md | 11 ++ .../zarr_metadata/rules/_entities/blosc.py | 115 ++++++++++++++++++ .../tests/rules/test_registry.py | 1 - .../tests/rules/test_v3_array_rules.py | 55 +++++++++ 4 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/changes/4379.feature.7.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md new file mode 100644 index 0000000000..a83a743107 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -0,0 +1,11 @@ +The `blosc` codec's values are judged: `clevel` in `[0, 9]`, `typesize` +positive, `blocksize` non-negative, and — the reason these live here +rather than in the entity's shape — `typesize` required unless `shuffle` +is `"noshuffle"`. + +That last one is the only conditionally-required member of any entity this +package models. A `TypedDict` cannot express it, so +`BloscCodecConfiguration` declares `typesize` as `NotRequired` and a rule +supplies the condition; without it, the member the spec singles out as +required would have been the one member of a blosc configuration that +could always be omitted. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py new file mode 100644 index 0000000000..6a918cf9ae --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py @@ -0,0 +1,115 @@ +"""Composition rules for the core `blosc` codec. + +`typesize` is the one configuration member of any entity this package +models whose requiredness the spec makes *conditional*: "Positive integer +specifying the stride in bytes over which shuffling is performed. Required +unless `shuffle` is `"noshuffle"`, in which case the value is ignored." + +A TypedDict cannot express that, so `BloscCodecConfiguration` declares it +`NotRequired` and the condition is enforced here. Without this rule the +member the spec singles out as required would be the one member of the +blosc configuration that could always be omitted, and the v3 changelog is +explicit that this is the parameter that must now be written down: "When +shuffling is enabled, the `typesize` must now be specified explicitly in +the metadata, rather than determined implicitly from the input data." + +The remaining rules judge the values the spec constrains and the shape +validator cannot: `clevel` "an integer from 0 to 9", `typesize` a +"positive integer", and `blocksize` a size in bytes, where "a value of 0 +indicates that an automatic size will be used" and a negative one names +nothing. + +https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import CODECS +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArrayParts + +_ARRAY_V3 = "zarr_v3_array" +_NO_SHUFFLE = "noshuffle" + + +@entity_rule( + _ARRAY_V3, + CODECS, + BLOSC_CODEC_NAME, + reads=frozenset({"shuffle"}), + reads_optional=frozenset({"typesize"}), +) +def typesize_is_present_when_shuffling( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, +) -> tuple[ValidationProblem, ...]: + """`typesize` is required unless `shuffle` is `"noshuffle"`.""" + shuffle = configuration["shuffle"] + if shuffle == _NO_SHUFFLE or "typesize" in configuration: + return () + return ( + ValidationProblem( + ("typesize",), + f"typesize is required when shuffle is {shuffle!r}", + "missing_key", + ), + ) + + +@entity_rule(_ARRAY_V3, CODECS, BLOSC_CODEC_NAME, reads_optional=frozenset({"typesize"})) +def typesize_is_positive( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, +) -> tuple[ValidationProblem, ...]: + if "typesize" not in configuration: + return () + typesize = cast("int", configuration["typesize"]) + if typesize >= 1: + return () + return ( + ValidationProblem( + ("typesize",), f"expected a positive integer, got {typesize}", "invalid_value" + ), + ) + + +@entity_rule(_ARRAY_V3, CODECS, BLOSC_CODEC_NAME, reads=frozenset({"clevel"})) +def clevel_is_in_range( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, +) -> tuple[ValidationProblem, ...]: + clevel = cast("int", configuration["clevel"]) + if 0 <= clevel <= 9: + return () + return ( + ValidationProblem( + ("clevel",), f"expected an integer in [0, 9], got {clevel}", "invalid_value" + ), + ) + + +@entity_rule(_ARRAY_V3, CODECS, BLOSC_CODEC_NAME, reads=frozenset({"blocksize"})) +def blocksize_is_non_negative( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, +) -> tuple[ValidationProblem, ...]: + blocksize = cast("int", configuration["blocksize"]) + if blocksize >= 0: + return () + return ( + ValidationProblem( + ("blocksize",), f"expected a non-negative integer, got {blocksize}", "invalid_value" + ), + ) diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py index 99450f1c47..910913d814 100644 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -48,7 +48,6 @@ # silent omission. _RULE_FREE = frozenset( { - (CODECS, "blosc"), (CODECS, "crc32c"), (CODECS, "scale_offset"), (CODECS, "zstd"), diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 2cc1455de1..cd044036ae 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -664,3 +664,58 @@ def test_error_a_malformed_must_understand_does_not_suppress_the_entity() -> Non } ) assert ("codecs", 0, "configuration", "order") in {problem.loc for problem in problems} + + +# -- blosc: the one member whose requiredness the spec makes conditional ------ + + +def _blosc_configuration(**overrides: object) -> dict[str, object]: + return { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "typesize": 2, + "blocksize": 0, + **overrides, + } + + +def _blosc(**overrides: object) -> Mapping[str, object]: + return {"name": "blosc", "configuration": _blosc_configuration(**overrides)} + + +def _with_blosc(**overrides: object) -> Mapping[str, object]: + return {**BASE, "codecs": ("bytes", _blosc(**overrides))} + + +def test_blosc_accepts_a_document_zarr_python_writes() -> None: + assert validate_array_metadata_v3(_with_blosc()) == () + + +def test_blosc_typesize_may_be_omitted_only_without_shuffling() -> None: + # "Required unless `shuffle` is `"noshuffle"`, in which case the value + # is ignored." A TypedDict cannot say that, so the rule does. + configuration = _blosc_configuration() + del configuration["typesize"] + for shuffle, required in (("noshuffle", False), ("shuffle", True), ("bitshuffle", True)): + codec = {"name": "blosc", "configuration": {**configuration, "shuffle": shuffle}} + problems = validate_array_metadata_v3({**BASE, "codecs": ("bytes", codec)}) + assert [problem.loc[-1] for problem in problems] == (["typesize"] if required else []) + + +def test_error_blosc_typesize_is_not_positive() -> None: + loc, message = _sole_problem(_with_blosc(typesize=0)) + assert loc == ("codecs", 1, "configuration", "typesize") + assert "positive" in message + + +def test_error_blosc_clevel_out_of_range() -> None: + loc, message = _sole_problem(_with_blosc(clevel=99)) + assert loc == ("codecs", 1, "configuration", "clevel") + assert "[0, 9]" in message + + +def test_error_blosc_blocksize_is_negative() -> None: + loc, message = _sole_problem(_with_blosc(blocksize=-1)) + assert loc == ("codecs", 1, "configuration", "blocksize") + assert "non-negative" in message From d43d28f843d9f5390bf386c4bb9133782dca7888 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 14:25:14 +0200 Subject: [PATCH 021/107] feat(zarr-metadata): judge the zstd compression level "An integer from -131072 to 22 which controls the speed and level of compression", with 0 selecting the default. The shape validator can only say it is an integer, so the range is a rule, and zstd leaves the deliberately-rule-free list alongside blosc. `checksum` needs none: the spec marks it "(Optional)" and the TypedDict already declares it `NotRequired`. Its "Should be omitted if false" is a SHOULD, and this package reports violations of requirements rather than of advice. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.7.md | 4 +- .../src/zarr_metadata/rules/_entities/zstd.py | 50 +++++++++++++++++++ .../tests/rules/test_registry.py | 1 - .../tests/rules/test_v3_array_rules.py | 17 +++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index a83a743107..71ca3d0d34 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -1,4 +1,4 @@ -The `blosc` codec's values are judged: `clevel` in `[0, 9]`, `typesize` +The compressors' values are judged. For `blosc`: `clevel` in `[0, 9]`, `typesize` positive, `blocksize` non-negative, and — the reason these live here rather than in the entity's shape — `typesize` required unless `shuffle` is `"noshuffle"`. @@ -9,3 +9,5 @@ package models. A `TypedDict` cannot express it, so supplies the condition; without it, the member the spec singles out as required would have been the one member of a blosc configuration that could always be omitted. + +For `zstd`, `level` must be in `[-131072, 22]`. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py new file mode 100644 index 0000000000..12395c7db2 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py @@ -0,0 +1,50 @@ +"""Composition rules for the core `zstd` codec. + +The spec gives `level` a range the shape validator cannot state: "An +integer from -131072 to 22 which controls the speed and level of +compression (has no impact on decoding). A value of 0 indicates to use the +default compression level." + +`checksum` needs no rule: the spec marks it "(Optional)" and the TypedDict +already declares it `NotRequired`. Its "Should be omitted if false" is a +SHOULD, and this package reports violations of requirements rather than of +advice. + +https://github.com/zarr-developers/zarr-extensions/blob/main/codecs/zstd/README.md +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.rules._registry import entity_rule +from zarr_metadata.v3._extension_points import CODECS +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.rules._spec import ArrayParts + +_ARRAY_V3 = "zarr_v3_array" +_MIN_LEVEL: Final = -131072 +_MAX_LEVEL: Final = 22 + + +@entity_rule(_ARRAY_V3, CODECS, ZSTD_CODEC_NAME, reads=frozenset({"level"})) +def level_is_in_range( + configuration: Mapping[str, object], + document: Mapping[str, object], + incoming: ArrayParts | None, +) -> tuple[ValidationProblem, ...]: + level = cast("int", configuration["level"]) + if _MIN_LEVEL <= level <= _MAX_LEVEL: + return () + return ( + ValidationProblem( + ("level",), + f"expected an integer in [{_MIN_LEVEL}, {_MAX_LEVEL}], got {level}", + "invalid_value", + ), + ) diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py index 910913d814..e8618139d8 100644 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -50,7 +50,6 @@ { (CODECS, "crc32c"), (CODECS, "scale_offset"), - (CODECS, "zstd"), (CHUNK_KEY_ENCODING, "default"), (CHUNK_KEY_ENCODING, "v2"), (DATA_TYPE, "bool"), diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index cd044036ae..9be104a0c5 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -719,3 +719,20 @@ def test_error_blosc_blocksize_is_negative() -> None: loc, message = _sole_problem(_with_blosc(blocksize=-1)) assert loc == ("codecs", 1, "configuration", "blocksize") assert "non-negative" in message + + +@pytest.mark.parametrize( + ("level", "valid"), + [(0, True), (22, True), (-131072, True), (23, False), (-131073, False), (1000, False)], +) +def test_zstd_level_range(level: int, valid: bool) -> None: + # "An integer from -131072 to 22"; 0 selects the default level. + document = { + **BASE, + "codecs": ("bytes", {"name": "zstd", "configuration": {"level": level, "checksum": False}}), + } + problems = validate_array_metadata_v3(document) + if valid: + assert problems == () + else: + assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "level")] From aa05d84073268764e90012e0ee0a17da7584bfa6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 15:51:45 +0200 Subject: [PATCH 022/107] refactor(zarr-metadata): value constraints belong with the type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most "rules" were not composition judgments at all. Of 23 entity rules, 15 read only the entity's own configuration: blosc's clevel range, gzip's and zstd's level ranges, a transpose order being a permutation of its own indices, a time scale factor's bounds. Those are refinements of the type — the TypedDict says int, the spec says int in [0, 9] — and they were in the rule layer because that is where the machinery happened to live, not because anything about them spans a document. They now sit where the member is typed. `_shape` gains a small value vocabulary (`_int_in_range`, positive and non-negative integers, a permutation checker) and, for the one constraint spanning two members of a single configuration, an entity invariant: blosc's "typesize is required unless shuffle is noshuffle". Seven rules become table entries, three rule modules are deleted outright, and `rules._entities` is left with the eight judgments that genuinely need the document or the chain. The boundary is the member, and the attempt to push past it is recorded because it failed usefully: moving per-element checks (every extent of a chunk_shape positive) into the shape layer cost real precision, because a shape verdict marks the whole member unusable. A zero on one axis stopped a rectilinear grid reporting the axis beside it and stood down a shard's inner pipeline. Those stay where they can be judged element by element. Document rules get the same layout: `_v3_array` is partitioned into field rules (codec pipeline ordering, known-entity shapes) and composition rules (fill value against data type, dimension names against shape), with a test asserting the partition so a new rule forces the decision the way `_RULE_FREE` already does for entities. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.7.md | 39 +++-- .../zarr_metadata/rules/_entities/blosc.py | 115 -------------- .../src/zarr_metadata/rules/_entities/gzip.py | 31 ---- .../rules/_entities/transpose.py | 28 +--- .../src/zarr_metadata/rules/_entities/zstd.py | 50 ------- .../src/zarr_metadata/rules/_v3_array.py | 32 +++- .../src/zarr_metadata/v3/_shape.py | 141 ++++++++++++++++-- .../src/zarr_metadata/v3/codec/blosc.py | 9 ++ .../tests/rules/test_registry.py | 67 ++++++++- 9 files changed, 258 insertions(+), 254 deletions(-) delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 71ca3d0d34..3c845ea767 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -1,13 +1,30 @@ -The compressors' values are judged. For `blosc`: `clevel` in `[0, 9]`, `typesize` -positive, `blocksize` non-negative, and — the reason these live here -rather than in the entity's shape — `typesize` required unless `shuffle` -is `"noshuffle"`. +Value constraints live with the type they refine, not in the rule layer. +The package now distinguishes three judgments rather than two: **type** +(is this an integer?), **value** (is it an integer in `[0, 9]`?), and +**composition** (does this codec's rank match the array that reached it?). +Only the third needs a document or a codec chain, and only the third is a +rule. -That last one is the only conditionally-required member of any entity this -package models. A `TypedDict` cannot express it, so -`BloscCodecConfiguration` declares `typesize` as `NotRequired` and a rule -supplies the condition; without it, the member the spec singles out as -required would have been the one member of a blosc configuration that -could always be omitted. +So `blosc`'s `clevel` range, `gzip`'s and `zstd`'s `level` ranges, +`typesize` being positive, `blocksize` being non-negative, a `transpose` +`order` being a permutation of its own indices, and a time data type's +`scale_factor` range are all stated where the member is typed, in +`v3._shape`. `blosc`'s "typesize is required unless shuffle is +`noshuffle`" spans two members of one configuration, so it is an entity +invariant there. Seven of the 23 entity rules became declarations, three +rule modules are gone, and what is left in `rules._entities` is the eight +judgments that genuinely read the document or the chain. -For `zstd`, `level` must be in `[-131072, 22]`. +The boundary is the *member*, for a reason worth recording: a verdict from +the shape layer marks a whole member unusable, so a constraint on the +elements *within* one — every extent of a `chunk_shape` being positive — +would cost precision elsewhere. A zero on one axis would stop a +rectilinear grid reporting the axis beside it, and would stand down a +shard's inner pipeline entirely. Those stay in the rule layer, which +judges element by element. + +Document-level rules follow the same layout: `_v3_array` is split into +field rules (one top-level field, such as codec pipeline ordering) and +composition rules (spanning fields, such as fill value against data type), +and a test asserts the partition so that adding a rule is a deliberate +choice about which it is. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py deleted file mode 100644 index 6a918cf9ae..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/blosc.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Composition rules for the core `blosc` codec. - -`typesize` is the one configuration member of any entity this package -models whose requiredness the spec makes *conditional*: "Positive integer -specifying the stride in bytes over which shuffling is performed. Required -unless `shuffle` is `"noshuffle"`, in which case the value is ignored." - -A TypedDict cannot express that, so `BloscCodecConfiguration` declares it -`NotRequired` and the condition is enforced here. Without this rule the -member the spec singles out as required would be the one member of the -blosc configuration that could always be omitted, and the v3 changelog is -explicit that this is the parameter that must now be written down: "When -shuffling is enabled, the `typesize` must now be specified explicitly in -the metadata, rather than determined implicitly from the input data." - -The remaining rules judge the values the spec constrains and the shape -validator cannot: `clevel` "an integer from 0 to 9", `typesize` a -"positive integer", and `blocksize` a size in bytes, where "a value of 0 -indicates that an automatic size will be used" and a negative one names -nothing. - -https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.v3._extension_points import CODECS -from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" -_NO_SHUFFLE = "noshuffle" - - -@entity_rule( - _ARRAY_V3, - CODECS, - BLOSC_CODEC_NAME, - reads=frozenset({"shuffle"}), - reads_optional=frozenset({"typesize"}), -) -def typesize_is_present_when_shuffling( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, -) -> tuple[ValidationProblem, ...]: - """`typesize` is required unless `shuffle` is `"noshuffle"`.""" - shuffle = configuration["shuffle"] - if shuffle == _NO_SHUFFLE or "typesize" in configuration: - return () - return ( - ValidationProblem( - ("typesize",), - f"typesize is required when shuffle is {shuffle!r}", - "missing_key", - ), - ) - - -@entity_rule(_ARRAY_V3, CODECS, BLOSC_CODEC_NAME, reads_optional=frozenset({"typesize"})) -def typesize_is_positive( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, -) -> tuple[ValidationProblem, ...]: - if "typesize" not in configuration: - return () - typesize = cast("int", configuration["typesize"]) - if typesize >= 1: - return () - return ( - ValidationProblem( - ("typesize",), f"expected a positive integer, got {typesize}", "invalid_value" - ), - ) - - -@entity_rule(_ARRAY_V3, CODECS, BLOSC_CODEC_NAME, reads=frozenset({"clevel"})) -def clevel_is_in_range( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, -) -> tuple[ValidationProblem, ...]: - clevel = cast("int", configuration["clevel"]) - if 0 <= clevel <= 9: - return () - return ( - ValidationProblem( - ("clevel",), f"expected an integer in [0, 9], got {clevel}", "invalid_value" - ), - ) - - -@entity_rule(_ARRAY_V3, CODECS, BLOSC_CODEC_NAME, reads=frozenset({"blocksize"})) -def blocksize_is_non_negative( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, -) -> tuple[ValidationProblem, ...]: - blocksize = cast("int", configuration["blocksize"]) - if blocksize >= 0: - return () - return ( - ValidationProblem( - ("blocksize",), f"expected a non-negative integer, got {blocksize}", "invalid_value" - ), - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py deleted file mode 100644 index b26744dbfc..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Composition rules for the core ``gzip`` codec.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.v3._extension_points import CODECS -from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" - - -@entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME, reads=frozenset({"level"})) -def level_is_in_range( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - level = cast("int", configuration["level"]) - if 0 <= level <= 9: - return () - return ( - ValidationProblem( - ("level",), f"expected an integer in [0, 9], got {level}", "invalid_value" - ), - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py index ee20e515e8..040c9403d3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py @@ -1,4 +1,9 @@ -"""Composition rules and spec transition for the `transpose` codec.""" +"""Composition rules and spec transition for the `transpose` codec. + +Whether `order` is a permutation of its own indices is a fact about the +value, checked by `v3._shape`. What is left here needs the array that +reached the codec. +""" from __future__ import annotations @@ -32,27 +37,6 @@ def permute_grid(configuration: Mapping[str, object], incoming: ArrayParts) -> A return incoming.with_grid(incoming.grid.permuted(order)) -@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) -def order_is_a_permutation( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """`order` must be a permutation of its own indices. - - Checked without reference to the incoming shape, so it holds even - when propagation has stopped upstream. - """ - order = cast("tuple[int, ...]", configuration["order"]) - if sorted(order) == list(range(len(order))): - return () - return ( - ValidationProblem( - ("order",), - f"expected a permutation of 0..{len(order) - 1}, got {order!r}", - "invalid_value", - ), - ) - - @entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) def order_matches_incoming_rank( configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py deleted file mode 100644 index 12395c7db2..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/zstd.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Composition rules for the core `zstd` codec. - -The spec gives `level` a range the shape validator cannot state: "An -integer from -131072 to 22 which controls the speed and level of -compression (has no impact on decoding). A value of 0 indicates to use the -default compression level." - -`checksum` needs no rule: the spec marks it "(Optional)" and the TypedDict -already declares it `NotRequired`. Its "Should be omitted if false" is a -SHOULD, and this package reports violations of requirements rather than of -advice. - -https://github.com/zarr-developers/zarr-extensions/blob/main/codecs/zstd/README.md -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Final, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.v3._extension_points import CODECS -from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" -_MIN_LEVEL: Final = -131072 -_MAX_LEVEL: Final = 22 - - -@entity_rule(_ARRAY_V3, CODECS, ZSTD_CODEC_NAME, reads=frozenset({"level"})) -def level_is_in_range( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, -) -> tuple[ValidationProblem, ...]: - level = cast("int", configuration["level"]) - if _MIN_LEVEL <= level <= _MAX_LEVEL: - return () - return ( - ValidationProblem( - ("level",), - f"expected an integer in [{_MIN_LEVEL}, {_MAX_LEVEL}], got {level}", - "invalid_value", - ), - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py index e012db51aa..8df9b1c398 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py @@ -1,4 +1,19 @@ -"""Composition rules for v3 array metadata documents. +"""Rules over a whole v3 array metadata document. + +The same three-way split the entity layer uses applies here, one level +up. **Type** is the document's structure, checked by +`zarr_metadata.model`. **Value** is a constraint on one field's own +content — the codec pipeline's kind ordering, a known entity's canonical +shape — which needs nothing else in the document to decide. **Composition** +is a judgment spanning fields: a fill value against its data type, one +dimension name per dimension of `shape`. + +A rule's `requires` says which it is, and the two are kept in separate +sections below; `tests/rules/test_registry.py` asserts the partition, so +adding a rule is a deliberate choice rather than an accident of where the +cursor was. Value rules over a single *entity* go one level further down, +into `v3._shape` if they constrain a member and into `rules._entities` if +they constrain elements within one. Whole-document rules live here: judgments that read several top-level fields, or that apply to a field regardless of which extension occupies @@ -299,7 +314,7 @@ def _struct_fill_problems( # --------------------------------------------------------------------------- -# whole-document rules +# field rules: each reads one top-level field and nothing else # --------------------------------------------------------------------------- ZARR_V3_ARRAY = "zarr_v3_array" @@ -310,9 +325,6 @@ def _struct_fill_problems( _data_type_spelling = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( _check_data_type_spelling ) -_fill_matches_dtype = document_rule(ZARR_V3_ARRAY, frozenset({"data_type", "fill_value"}))( - _check_fill_matches_dtype -) def _known_entity_shape( @@ -329,6 +341,7 @@ def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: found = validate_known_entity_metadata(field, document[field]) return () if found is None else prefixed((field,), found) + check.__name__ = f"_check_{field}_shape" return check @@ -368,6 +381,15 @@ def check_chunk_grid_shape(document: Mapping[str, object]) -> tuple[ValidationPr return prefixed(("chunk_grid",), found) +# --------------------------------------------------------------------------- +# composition rules: each spans more than one top-level field +# --------------------------------------------------------------------------- + +_fill_matches_dtype = document_rule(ZARR_V3_ARRAY, frozenset({"data_type", "fill_value"}))( + _check_fill_matches_dtype +) + + @document_rule(ZARR_V3_ARRAY, frozenset({"shape", "dimension_names"})) def check_dimension_names_length(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: """One dimension name per array dimension.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py index 454bd44b76..8a94ab7981 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py @@ -15,8 +15,29 @@ the TypedDicts declare. Normalize a freshly-`json.loads`-ed document (e.g. with a model-layer parser) before asking for shape verdicts. -Value judgments beyond the types — permutation contents, shard geometry, -cross-field consistency — belong to the composition rule layer, not here. +Three kinds of judgment, and this module owns the first two: + +- **type**: is this an integer? — the TypedDicts, checked member by member. +- **value**: is it an integer *in [0, 9]*? — a constraint the spec places on + one member's value, or on one entity's own configuration. Stated here as + a richer checker, or as an entity `invariant` when it spans two members + of the same configuration, because it is a refinement of the type and + needs nothing outside the entity to decide. +- **composition**: does this codec's rank match the array that reached it? + — needs the document or the codec chain, and belongs to + `zarr_metadata.rules`. + +Putting value constraints here rather than in the rule layer keeps the +answer next to the type it refines, and means a rule only exists where a +judgment genuinely spans more than one entity. + +The boundary is the *member*, and it is not a matter of taste. A verdict +here marks a whole member unusable, so a constraint on the elements +*within* one — every extent of a `chunk_shape` being positive — would cost +precision elsewhere: a zero on one axis would stop a rectilinear grid +reporting the axis beside it, and would stand down a shard's inner +pipeline entirely. Those stay in the rule layer, which judges element by +element. Constraints on a member as a whole belong here. Unknown names are not judged (extension openness): the `validate_known_*` functions answer `None` for entities this package has no types for, no @@ -33,7 +54,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, cast @@ -76,6 +97,7 @@ from zarr_metadata.v3.codec.blosc import ( BLOSC_CNAME, BLOSC_CODEC_NAME, + BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE, BloscCodecConfiguration, BloscCodecObject, @@ -148,11 +170,19 @@ from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME if TYPE_CHECKING: - from collections.abc import Callable - from zarr_metadata.model._validation import ProblemKind - _FieldChecker = Callable[[object, tuple[str | int, ...]], tuple[ValidationProblem, ...]] +_FieldChecker = Callable[[object, tuple[str | int, ...]], tuple[ValidationProblem, ...]] +"""A constraint on one configuration member's value, given its location.""" + +_EntityInvariant = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] +"""A value constraint spanning two members of one configuration. + +Runs only once every member has passed its own checker, so it may read +them without guarding; `blosc`'s "typesize is required unless shuffle is +noshuffle" is the whole population today. Locations are relative to the +configuration. +""" def entity_name(value: object) -> str | None: @@ -188,6 +218,73 @@ def _check_json_bool(value: object, loc: tuple[str | int, ...]) -> tuple[Validat return () +def _int_in_range(low: int, high: int) -> _FieldChecker: + """An integer the spec confines to `[low, high]`.""" + + def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + problems = _check_json_int(value, loc) + if len(problems) != 0: + return problems + if low <= cast("int", value) <= high: + return () + return _problems( + loc, f"expected an integer in [{low}, {high}], got {value!r}", "invalid_value" + ) + + return check + + +def _bounded_int(low: int, description: str) -> _FieldChecker: + """An integer the spec bounds from below only.""" + + def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + problems = _check_json_int(value, loc) + if len(problems) != 0: + return problems + if cast("int", value) >= low: + return () + return _problems(loc, f"expected {description}, got {value!r}", "invalid_value") + + return check + + +_check_positive_int = _bounded_int(1, "a positive integer") +_check_non_negative_int = _bounded_int(0, "a non-negative integer") + + +def _check_permutation(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: + """A transpose order: a permutation of its own indices. + + Whether it also matches the rank of the array that reached the codec + is composition, and lives in `rules._entities.transpose`. + """ + problems = _check_int_tuple(value, loc) + if len(problems) != 0: + return problems + order = cast("tuple[int, ...]", value) + if sorted(order) == list(range(len(order))): + return () + return _problems( + loc, f"expected a permutation of 0..{len(order) - 1}, got {order!r}", "invalid_value" + ) + + +def _blosc_typesize_is_present_when_shuffling( + configuration: Mapping[str, object], +) -> tuple[ValidationProblem, ...]: + """`typesize` is required unless `shuffle` is `"noshuffle"`. + + The one constraint in this package that spans two members of a single + configuration, and the reason `_EntityShape` carries invariants at all. + """ + shuffle = configuration.get("shuffle") + if shuffle == BLOSC_NO_SHUFFLE or "typesize" in configuration: + return () + return _problems( + ("typesize",), f"typesize is required when shuffle is {shuffle!r}", "missing_key" + ) + + def _literal(allowed: tuple[str, ...]) -> _FieldChecker: def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: if value not in allowed: @@ -368,12 +465,14 @@ class _EntityShape: config_keys: frozenset[str] config_required: frozenset[str] config_checkers: Mapping[str, _FieldChecker] + invariants: tuple[_EntityInvariant, ...] = () def _shape( object_type: type, configuration_type: type, checkers: Mapping[str, _FieldChecker], + invariants: tuple[_EntityInvariant, ...] = (), ) -> _EntityShape: config_keys = frozenset(configuration_type.__annotations__) if frozenset(checkers) != config_keys: @@ -390,6 +489,7 @@ def _shape( configuration_type.__required_keys__, # type: ignore[attr-defined] ), config_checkers=dict(checkers), + invariants=invariants, ) @@ -419,11 +519,12 @@ def _bare_shape() -> _EntityShape: BloscCodecConfiguration, { "cname": _literal(BLOSC_CNAME), - "clevel": _check_json_int, + "clevel": _int_in_range(0, 9), "shuffle": _literal(BLOSC_SHUFFLE), - "blocksize": _check_json_int, - "typesize": _check_json_int, + "blocksize": _check_non_negative_int, + "typesize": _check_positive_int, }, + invariants=(_blosc_typesize_is_present_when_shuffling,), ), BYTES_CODEC_NAME: _shape( BytesCodecObject, BytesCodecConfiguration, {"endian": _literal(ENDIANNESS)} @@ -439,7 +540,9 @@ def _bare_shape() -> _EntityShape: }, ), CRC32C_CODEC_NAME: _shape(Crc32cCodecObject, Empty, {}), - GZIP_CODEC_NAME: _shape(GzipCodecObject, GzipCodecConfiguration, {"level": _check_json_int}), + GZIP_CODEC_NAME: _shape( + GzipCodecObject, GzipCodecConfiguration, {"level": _int_in_range(0, 9)} + ), SCALE_OFFSET_CODEC_NAME: _shape( ScaleOffsetCodecObject, ScaleOffsetCodecConfiguration, @@ -456,12 +559,12 @@ def _bare_shape() -> _EntityShape: }, ), TRANSPOSE_CODEC_NAME: _shape( - TransposeCodecObject, TransposeCodecConfiguration, {"order": _check_int_tuple} + TransposeCodecObject, TransposeCodecConfiguration, {"order": _check_permutation} ), ZSTD_CODEC_NAME: _shape( ZstdCodecObject, ZstdCodecConfiguration, - {"level": _check_json_int, "checksum": _check_json_bool}, + {"level": _int_in_range(-131072, 22), "checksum": _check_json_bool}, ), } @@ -519,12 +622,12 @@ def _bare_shape() -> _EntityShape: NUMPY_DATETIME64_DATA_TYPE_NAME: _shape( NumpyDatetime64, NumpyDatetime64Configuration, - {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _check_json_int}, + {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _int_in_range(1, 2**31 - 1)}, ), NUMPY_TIMEDELTA64_DATA_TYPE_NAME: _shape( NumpyTimedelta64, NumpyTimedelta64Configuration, - {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _check_json_int}, + {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _int_in_range(1, 2**31 - 1)}, ), STRUCT_DATA_TYPE_NAME: _shape(Struct, StructConfiguration, {"fields": _check_struct_fields}), } @@ -585,6 +688,16 @@ def _validate_known_entity( for key, checker in shape.config_checkers.items(): if key in config: problems.extend(checker(config[key], ("configuration", key))) + if len(problems) == 0: + # Invariants read members directly, so they run only once every + # member has been vouched for; a complaint about one member is + # reason enough not to reason across them. + usable = cast("Mapping[str, object]", config) + for invariant in shape.invariants: + problems.extend( + ValidationProblem(("configuration", *found.loc), found.message, found.kind) + for found in invariant(usable) + ) return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 3387a7c285..9b7f266238 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -20,6 +20,14 @@ BLOSC_SHUFFLE: Final = ("noshuffle", "shuffle", "bitshuffle") """Tuple of permitted values for the `shuffle` field of the `blosc` codec.""" +BLOSC_NO_SHUFFLE: Final = "noshuffle" +"""The `shuffle` value under which `typesize` carries no information. + +The spec requires `typesize` "unless `shuffle` is `"noshuffle"`, in which +case the value is ignored", so this is the one value that changes whether +another member is required. +""" + BloscCName = Literal["lz4", "lz4hc", "blosclz", "snappy", "zlib", "zstd"] """Literal type of blosc compressor identifiers.""" @@ -58,6 +66,7 @@ class BloscCodecObject(TypedDict, closed=True): __all__ = [ "BLOSC_CNAME", "BLOSC_CODEC_NAME", + "BLOSC_NO_SHUFFLE", "BLOSC_SHUFFLE", "BloscCName", "BloscCodecConfiguration", diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py index e8618139d8..f333a550ec 100644 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ b/packages/zarr-metadata/tests/rules/test_registry.py @@ -15,7 +15,12 @@ import pytest import zarr_metadata.rules._entities as entities -from zarr_metadata.rules import ZARR_V2_ARRAY_RULES, ZARR_V3_ARRAY_RULES, ZARR_V3_GROUP_RULES +from zarr_metadata.rules import ( + ZARR_V2_ARRAY_RULES, + ZARR_V3_ARRAY_RULES, + ZARR_V3_GROUP_RULES, + Rule, +) from zarr_metadata.rules._registry import ( dispatched_fields, document_rule, @@ -41,15 +46,20 @@ from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME -# Entities the package models but that carry no composition rules: their -# canonical shape is the whole of what we can say about them. Listed by -# hand, keyed by extension point, so that adding a codec is a deliberate -# choice between "write rules" and "record that there are none", never a -# silent omission. +# Entities the package models that carry no *composition* rule — nothing +# about them depends on the document or on the codec chain. Several still +# have value constraints (`blosc`'s clevel range, `gzip`'s and `zstd`'s +# level ranges); those are refinements of the type and live with it in +# `v3._shape`, not here. Listed by hand, keyed by extension point, so that +# adding a codec is a deliberate choice between "write a rule" and "record +# that composition says nothing", never a silent omission. _RULE_FREE = frozenset( { + (CODECS, "blosc"), (CODECS, "crc32c"), + (CODECS, "gzip"), (CODECS, "scale_offset"), + (CODECS, "zstd"), (CHUNK_KEY_ENCODING, "default"), (CHUNK_KEY_ENCODING, "v2"), (DATA_TYPE, "bool"), @@ -214,3 +224,48 @@ def _subscripts_an_optional_member( incoming: ArrayParts | None, ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered return () + + +# Every document rule, by the layer it belongs to. A field rule reads one +# top-level field; a composition rule spans several. Listed by hand so that +# adding one is a deliberate choice, the way `_RULE_FREE` makes "this entity +# has no composition rule" a deliberate choice. +_FIELD_RULES = frozenset( + { + "_check_data_type_spelling", + "_check_data_type_shape", + "_check_chunk_key_encoding_shape", + "_check_chunk_grid_shape", + "check_codec_pipeline_order", + "check_codec_shapes", + "check_chunk_grid_shape", + "_dispatch_chunk_grid_entity_rules", + "_dispatch_data_type_entity_rules", + "_dispatch_chunk_key_encoding_entity_rules", + "_dispatch_codecs_entity_rules", + "check_consolidated_entries", + } +) +_COMPOSITION_RULES = frozenset( + {"_check_fill_matches_dtype", "check_dimension_names_length", "check_chunks_match_shape"} +) + + +@pytest.mark.parametrize( + "rules", + [ZARR_V3_ARRAY_RULES, ZARR_V2_ARRAY_RULES, ZARR_V3_GROUP_RULES], + ids=["v3-array", "v2-array", "v3-group"], +) +def test_document_rules_are_classified_by_what_they_read(rules: tuple[Rule, ...]) -> None: + # The classification is not decoration: a rule reading one field is a + # value constraint on that field, and could in principle move down to + # the layer that owns the field. One spanning fields cannot. + for rule in rules: + name = rule.check.__name__ + assert name in _FIELD_RULES | _COMPOSITION_RULES, f"{name} is classified nowhere" + if name in _FIELD_RULES: + assert len(rule.requires) == 1, ( + f"{name} is a field rule but reads {sorted(rule.requires)}" + ) + else: + assert len(rule.requires) >= 2, f"{name} is a composition rule but reads one field" From c0600eef8ea32105f9301459bd6cc5c82bfc54be Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 16:21:28 +0200 Subject: [PATCH 023/107] feat(zarr-metadata): canonicalize a document, or report why it cannot be MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One function from a syntactically valid document to either that document in its simplest equivalent spelling or every reason it is not semantically valid. Canonical is defined per metadata variety as the simplest form with the same semantics. The generic half — an entity with nothing to configure collapsing to its bare name, a defaulted `must_understand` dropping while an explicit `false` survives — the model layer's round trip already performs exactly, so it is delegated rather than written twice. What is new is per-variety: blosc drops a `typesize` the spec says is ignored under `noshuffle`, and a rectilinear dimension run-length encodes, since `[size, count]` is the spelling that does not grow with the chunk count. `dimension_names` of nothing but nulls says what omitting it says. Each canonicalization lives with the entity it belongs to, in `v3/codec/blosc.py` and `v3/chunk_grid/rectilinear.py`, because they are pure transforms over already-valid metadata and need nothing from the rules layer. Two spellings that look collapsible are left alone on purpose, and are tested: a dimension-level bare integer repeats until it covers the extent rather than naming a fixed list, and a one-element list names exactly one chunk. They produce identical extents but different coverage verdicts, so collapsing either would change meaning under a resize. Asserted over the generated corpus: canonicalizing twice changes nothing further, and canonicalizing never changes a verdict — the second is what would catch a simplification that quietly says something else. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.8.md | 22 +++ .../src/zarr_metadata/rules/__init__.py | 12 +- .../src/zarr_metadata/rules/_canonical.py | 142 ++++++++++++++++ .../v3/chunk_grid/rectilinear.py | 42 +++++ .../src/zarr_metadata/v3/codec/blosc.py | 18 ++ .../tests/rules/test_canonical.py | 155 ++++++++++++++++++ .../zarr-metadata/tests/test_public_api.py | 2 + 7 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/changes/4379.feature.8.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py create mode 100644 packages/zarr-metadata/tests/rules/test_canonical.py diff --git a/packages/zarr-metadata/changes/4379.feature.8.md b/packages/zarr-metadata/changes/4379.feature.8.md new file mode 100644 index 0000000000..cac06d2431 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.feature.8.md @@ -0,0 +1,22 @@ +Added `canonicalize_array_metadata_v3`, which answers with +`Canonical[T] | Invalid`: a semantically valid document in its simplest +equivalent spelling, or every reason it is not valid. + +Canonical is decided per metadata variety, and always means the simplest +form expressing the same semantics. An entity whose configuration carries +nothing collapses to its bare name, and a `must_understand` of `true` — +the default — is dropped while an explicit `false` is kept. `blosc` drops +a `typesize` that `shuffle: "noshuffle"` renders ignored, the spec saying +of that case that "the value is ignored". A rectilinear dimension's chunk +sizes run-length encode, because `[size, count]` is the spelling that does +not grow with the number of chunks. `dimension_names` of nothing but nulls +says what omitting the field says. + +Two spellings that look collapsible are deliberately left alone: a +dimension-level bare integer is a *step* that repeats until it covers the +extent, so it is not equivalent to any fixed list, and a one-element list +declares exactly one chunk rather than as many as it takes. Expanding +either would pin a grid that currently adapts. + +Two properties are asserted over generated documents: canonicalizing twice +changes nothing further, and canonicalizing never changes a verdict. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py index 80fc4ee3d1..52cc0ce221 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py @@ -3,7 +3,9 @@ `zarr_metadata.model` checks JSON structure. This module also checks cross-field constraints such as fill-value compatibility, codec ordering, and dimension counts. Its `validate_*` and `parse_*` functions -mirror the model API. +mirror the model API, and `canonicalize_array_metadata_v3` answers with +either the document in its simplest equivalent spelling or every reason +it is not valid. Rules target canonical metadata and may be stricter than readers that coerce inputs. Unknown entity names are left unjudged. Known entities @@ -12,6 +14,11 @@ round-trips preserve those unmodeled members. """ +from zarr_metadata.rules._canonical import ( + Canonical, + Invalid, + canonicalize_array_metadata_v3, +) from zarr_metadata.rules._documents import ( parse_array_metadata_v2, parse_array_metadata_v3, @@ -34,9 +41,12 @@ "ZARR_V3_ARRAY_RULES", "ZARR_V3_GROUP", "ZARR_V3_GROUP_RULES", + "Canonical", + "Invalid", "Rule", "RuleCheck", "applicable", + "canonicalize_array_metadata_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", "parse_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py new file mode 100644 index 0000000000..269db46653 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py @@ -0,0 +1,142 @@ +"""One document in, one canonical document or one report of why not. + +`canonicalize_array_metadata_v3` takes a *syntactically* valid document — +one the model layer has already accepted, so every member is present and +typed as its TypedDict declares — and answers with `Canonical[T] | Invalid`: either the same document in +canonical form or every reason it is not semantically valid. Testing the +literal `valid` field narrows to one or the other. + +Canonical means the simplest spelling with the same meaning, decided per +metadata variety: + +- an entity whose configuration carries nothing collapses to its bare + name, and a `must_understand` of `true` (the default) is dropped while + an explicit `false` is kept, because that one says something. This part + the model layer already performs, so it is delegated rather than + reimplemented. +- `blosc` drops a `typesize` that `shuffle: "noshuffle"` renders ignored. +- a rectilinear dimension's chunk sizes run-length encode, because that + is the spelling that does not grow with the number of chunks. +- `dimension_names` of nothing but nulls says what omitting the field + says. + +Two properties are worth holding on to, and +`tests/rules/test_canonical.py` asserts both: canonicalizing twice +changes nothing further, and canonicalizing never changes a verdict. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast + +from zarr_metadata.model._array import ZarrV3ArrayMetadata +from zarr_metadata.rules._documents import validate_array_metadata_v3 +from zarr_metadata.v3._shape import entity_name +from zarr_metadata.v3.chunk_grid.rectilinear import ( + RECTILINEAR_CHUNK_GRID_NAME, + canonical_chunk_shapes, +) +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME +from zarr_metadata.v3.codec.blosc import canonical_configuration as canonical_blosc + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ValidationProblem + from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON + from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearDimSpec + +DocumentT = TypeVar("DocumentT") + + +@dataclass(frozen=True, slots=True) +class Canonical(Generic[DocumentT]): + """A semantically valid document, in its simplest equivalent spelling.""" + + document: DocumentT + valid: Literal[True] = True + + +@dataclass(frozen=True, slots=True) +class Invalid: + """Every reason a document is not semantically valid; never empty.""" + + problems: tuple[ValidationProblem, ...] + valid: Literal[False] = False + + def __post_init__(self) -> None: + if len(self.problems) == 0: + msg = "Invalid requires at least one validation problem" + raise ValueError(msg) + + +def _canonical_entity(value: object) -> object: + """One entity's configuration in its simplest equivalent form. + + Only varieties with something to say appear here; everything else is + handed to the generic collapse unchanged. + """ + original: object = value + name = entity_name(value) + if name is None or not isinstance(value, Mapping): + return original + entry: Mapping[str, object] = cast("Mapping[str, object]", value) + configuration = entry.get("configuration") + if not isinstance(configuration, Mapping): + return original + members: Mapping[str, object] = cast("Mapping[str, object]", configuration) + if name == BLOSC_CODEC_NAME: + members = canonical_blosc(members) + elif name == RECTILINEAR_CHUNK_GRID_NAME: + shapes = members.get("chunk_shapes") + if isinstance(shapes, tuple): + specs = cast("tuple[RectilinearDimSpec, ...]", shapes) + members = {**members, "chunk_shapes": canonical_chunk_shapes(specs)} + if members is configuration: + return original + return {**entry, "configuration": members} + + +def _canonical_document(document: Mapping[str, object]) -> dict[str, object]: + """Per-variety canonicalization, before the generic collapse.""" + out = dict(document) + for field in ("chunk_grid", "chunk_key_encoding", "data_type"): + if field in out: + out[field] = _canonical_entity(out[field]) + codecs = out.get("codecs") + if isinstance(codecs, tuple): + entries = cast("tuple[object, ...]", codecs) + out["codecs"] = tuple(_canonical_entity(codec) for codec in entries) + names = out.get("dimension_names") + if isinstance(names, tuple) and all( + entry is None for entry in cast("tuple[object, ...]", names) + ): + # Every dimension unnamed says what saying nothing says. + del out["dimension_names"] + return out + + +def canonicalize_array_metadata_v3( + document: ZarrV3ArrayMetadataJSON, +) -> Canonical[ZarrV3ArrayMetadataJSON] | Invalid: + """`document` in canonical form, or every reason it is not valid. + + Expects a document the model layer has already accepted. Passing one + it has not is not an error — the composition problems are reported the + same way — but the structural problems come back too, and the result + is `Invalid` rather than a canonical document. + """ + problems = validate_array_metadata_v3(document) + if len(problems) != 0: + return Invalid(problems) + canonical = _canonical_document(document) + # The generic collapse — shorthand names, defaulted `must_understand` — + # is what the model layer's round trip already performs. + return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) + + +__all__ = [ + "Canonical", + "Invalid", + "canonicalize_array_metadata_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 78c38b702a..be2332fa86 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -47,6 +47,46 @@ class RectilinearChunkGridObject(TypedDict, closed=True): https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ + +def canonical_dim_spec(spec: RectilinearDimSpec) -> RectilinearDimSpec: + """One dimension's chunk sizes in their simplest equivalent form. + + Runs of equal sizes collapse to `[size, count]` pairs, because that is + the spelling that does not grow with the number of chunks: a million + equal chunks is two numbers, not a million. A run of one stays a bare + size, and `[size, 1]` collapses to one, since a pair says nothing extra + there. Adjacent spellings of the same size merge, which is what makes + this idempotent: `[[32, 2], 32]` and `[32, [32, 2]]` both become + `[[32, 3]]`. + + A dimension-level bare integer is left alone. It is a *step* that + repeats until it covers the extent, so it is not equivalent to any + fixed list — expanding it would pin a grid that currently adapts, and + the two would diverge the moment the array were resized. For the same + reason a one-element list is never collapsed to a bare integer: + `[32]` declares exactly one chunk and `32` declares as many as it takes. + + Assumes a spec the shape validator has already accepted. + """ + if not isinstance(spec, tuple): + return spec + runs: list[tuple[int, int]] = [] + for entry in spec: + size, count = entry if isinstance(entry, tuple) else (entry, 1) + if len(runs) != 0 and runs[-1][0] == size: + runs[-1] = (size, runs[-1][1] + count) + else: + runs.append((size, count)) + return tuple(size if count == 1 else (size, count) for size, count in runs) + + +def canonical_chunk_shapes( + chunk_shapes: tuple[RectilinearDimSpec, ...], +) -> tuple[RectilinearDimSpec, ...]: + """Every dimension's chunk sizes in their simplest equivalent form.""" + return tuple(canonical_dim_spec(spec) for spec in chunk_shapes) + + __all__ = [ "RECTILINEAR_CHUNK_GRID_NAME", "RectilinearChunkGridConfiguration", @@ -54,4 +94,6 @@ class RectilinearChunkGridObject(TypedDict, closed=True): "RectilinearChunkGridName", "RectilinearChunkGridObject", "RectilinearDimSpec", + "canonical_chunk_shapes", + "canonical_dim_spec", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 9b7f266238..5cce93fdbf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -4,6 +4,7 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html """ +from collections.abc import Mapping from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -63,6 +64,22 @@ class BloscCodecObject(TypedDict, closed=True): https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ + +def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, object]: + """A blosc configuration in its simplest equivalent form. + + Under `shuffle: "noshuffle"` the spec says of `typesize` that "the + value is ignored", so whatever it holds carries no meaning and two + documents differing only there describe the same codec. Dropping it + makes that equality visible. + + Assumes a configuration the shape validator has already accepted. + """ + if configuration.get("shuffle") != BLOSC_NO_SHUFFLE or "typesize" not in configuration: + return configuration + return {key: value for key, value in configuration.items() if key != "typesize"} + + __all__ = [ "BLOSC_CNAME", "BLOSC_CODEC_NAME", @@ -74,4 +91,5 @@ class BloscCodecObject(TypedDict, closed=True): "BloscCodecName", "BloscCodecObject", "BloscShuffle", + "canonical_configuration", ] diff --git a/packages/zarr-metadata/tests/rules/test_canonical.py b/packages/zarr-metadata/tests/rules/test_canonical.py new file mode 100644 index 0000000000..04b6b61957 --- /dev/null +++ b/packages/zarr-metadata/tests/rules/test_canonical.py @@ -0,0 +1,155 @@ +"""Canonicalization: the simplest spelling with the same meaning. + +Two properties carry the weight. Canonicalizing twice must change nothing +further, or the form is not canonical. And canonicalizing must never +change a verdict, or it is not meaning-preserving — that one is what +catches a "simplification" that quietly says something else. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from hypothesis import HealthCheck, given, settings + +from tests.rules.strategies import valid_documents +from zarr_metadata.rules import validate_array_metadata_v3 +from zarr_metadata.rules._canonical import Canonical, Invalid, canonicalize_array_metadata_v3 + +if TYPE_CHECKING: + from collections.abc import Mapping + +_SLOW = settings(max_examples=300, deadline=None, suppress_health_check=(HealthCheck.too_slow,)) + +BASE: Mapping[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (64,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (32,)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), +} + + +def _canonical(**overrides: object) -> Mapping[str, object]: + result = canonicalize_array_metadata_v3({**BASE, **overrides}) # type: ignore[arg-type] + assert isinstance(result, Canonical), result + return result.document + + +# (what it should simplify, the field it lands in, the expected canonical value) +SIMPLIFICATIONS: dict[str, tuple[dict[str, object], str, object]] = { + "object-with-no-configuration": ({"codecs": ({"name": "bytes"},)}, "codecs", ("bytes",)), + "object-with-empty-configuration": ( + {"codecs": ({"name": "bytes", "configuration": {}},)}, + "codecs", + ("bytes",), + ), + "defaulted-must-understand": ( + {"codecs": ({"name": "bytes", "must_understand": True},)}, + "codecs", + ("bytes",), + ), + "chunk-key-encoding-shorthand": ( + {"chunk_key_encoding": {"name": "default"}}, + "chunk_key_encoding", + "default", + ), + "rectilinear-runs-encode": ( + { + "chunk_grid": { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((32, 32),)}, + } + }, + "chunk_grid", + { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (((32, 2),),)}, + }, + ), +} + + +@pytest.mark.parametrize( + ("overrides", "field", "expected"), SIMPLIFICATIONS.values(), ids=list(SIMPLIFICATIONS) +) +def test_simplifies(overrides: dict[str, object], field: str, expected: object) -> None: + assert _canonical(**overrides)[field] == expected + + +def test_an_explicit_must_understand_false_is_kept() -> None: + # `true` is the default and says nothing; `false` says something. + codec = {"name": "bytes", "must_understand": False} + assert _canonical(codecs=(codec,))["codecs"] == (codec,) + + +def test_blosc_drops_a_typesize_that_shuffle_renders_ignored() -> None: + configuration = {"cname": "zstd", "clevel": 5, "blocksize": 0, "typesize": 4} + dropped = _canonical( + codecs=( + "bytes", + {"name": "blosc", "configuration": {**configuration, "shuffle": "noshuffle"}}, + ) + )["codecs"][1] # type: ignore[index] + assert "typesize" not in dropped["configuration"] # type: ignore[index] + kept = _canonical( + codecs=( + "bytes", + {"name": "blosc", "configuration": {**configuration, "shuffle": "shuffle"}}, + ) + )["codecs"][1] # type: ignore[index] + assert kept["configuration"]["typesize"] == 4 # type: ignore[index] + + +def test_dimension_names_of_nothing_but_nulls_are_dropped() -> None: + assert "dimension_names" not in _canonical(dimension_names=(None,)) + assert _canonical(dimension_names=("x",))["dimension_names"] == ("x",) + + +def test_a_rectilinear_step_is_not_expanded() -> None: + # A bare integer repeats to cover the extent, so it is not equivalent + # to any fixed list -- and a one-element list is not equivalent to it. + for spec in (32, (32,)): + grid = { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (spec,)}, + } + shape = (64,) if spec == 32 else (32,) + result = canonicalize_array_metadata_v3({**BASE, "shape": shape, "chunk_grid": grid}) # type: ignore[arg-type] + assert isinstance(result, Canonical), result + assert result.document["chunk_grid"]["configuration"]["chunk_shapes"] == (spec,) # type: ignore[index] + + +def test_error_a_semantically_invalid_document_reports_instead() -> None: + result = canonicalize_array_metadata_v3({**BASE, "fill_value": 999}) # type: ignore[arg-type] + assert isinstance(result, Invalid) + assert any("fill_value" in problem.message for problem in result.problems) + + +def test_error_invalid_cannot_be_empty() -> None: + with pytest.raises(ValueError, match="at least one"): + Invalid(()) + + +@given(valid_documents()) +@_SLOW +def test_canonicalizing_twice_changes_nothing_further(doc: Mapping[str, object]) -> None: + once = canonicalize_array_metadata_v3(doc) # type: ignore[arg-type] + assert isinstance(once, Canonical), once + twice = canonicalize_array_metadata_v3(once.document) + assert isinstance(twice, Canonical), twice + assert twice.document == once.document + + +@given(valid_documents()) +@_SLOW +def test_canonicalizing_never_changes_the_verdict(doc: Mapping[str, object]) -> None: + # A simplification that changed meaning would show up here as a + # document that validated before and does not after. + result = canonicalize_array_metadata_v3(doc) # type: ignore[arg-type] + assert isinstance(result, Canonical), result + assert validate_array_metadata_v3(result.document) == () diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index ca880c0070..fafa67ff59 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -279,10 +279,12 @@ def test_all_is_grouped_and_unique() -> None: "Base64Bytes", "BloscCName", "BloscShuffle", + "Canonical", "CastOutOfRangeMode", "CastRoundingMode", "CodecKind", "Endianness", + "Invalid", "HexFloat16", "HexFloat32", "HexFloat64", From 3dec64f838f4810bd2ff02cd47608c7560484714 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 16:44:27 +0200 Subject: [PATCH 024/107] feat(zarr-metadata): an entity knows its own shape, values and spelling A codec's knowledge was spread over four modules: a member-type table in the shape layer, a value-rule module in the rules layer, a kind tuple, a canonicalizer. Four tables meant four chances to drift, and a drift test per table to notice. Put it in the class instead. `coerce` is type-space, `problems` is value-space, `to_json` is the canonical spelling, and the dataclass fields are the configuration -- so the constructor signature is the configuration TypedDict unpacked, and a test asserts it stays that way. `Context` is the scope a reading happens in: which identifiers are in play at each extension point, as either the core specification or the core plus the extensions this package models. It is passed to every `coerce` and most ignore it, but `struct` and `sharding_indexed` carry other entities inside their configuration and cannot coerce without it. blosc is the first entity through; the rest follow. Nothing is wired in yet, so the existing path is untouched. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 275 ++++++++++++++++++ .../src/zarr_metadata/v3/_registry.py | 75 +++++ .../src/zarr_metadata/v3/codec/blosc.py | 131 ++++++++- .../zarr-metadata/tests/v3/test_entities.py | 73 +++++ 4 files changed, 551 insertions(+), 3 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_entity.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_registry.py create mode 100644 packages/zarr-metadata/tests/v3/test_entities.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py new file mode 100644 index 0000000000..30f5b74b4d --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -0,0 +1,275 @@ +"""What every metadata entity can do for itself. + +A codec, data type, chunk grid or chunk key encoding is three things at +once: a JSON shape, a set of constraints on the values in that shape, and +a canonical spelling. Keeping the three apart put one entity's knowledge +in four modules and needed a table per axis plus a drift test per table to +hold them together. Here they are one class per entity, and the class is +where methods bind: + +- `coerce` is **type-space**: raw metadata in, the entity or the reasons + it is not that entity out. +- `problems` is **value-space**: the entity is well-typed by construction, + so this only asks whether its values are in range. +- `to_json` is **canonical**: the simplest spelling meaning the same, typed + as the entity's own object TypedDict. + +The TypedDicts stay: they model the JSON form, and the correspondence is +exact in both directions. A configuration TypedDict unpacked is the +dataclass constructor's signature, and the object TypedDict is what +`to_json` returns. `tests/v3/test_entities.py` asserts the first, so the +two cannot drift. + +Everything that needs the document or the codec chain stays outside, in +`zarr_metadata.rules`, because an entity cannot answer it alone. + +`coerce` takes a `Context`: the entities in scope for this reading. Most +entities ignore it -- a `gzip` codec is a `gzip` codec whatever else is +registered -- but the ones whose configuration contains other entities do +not. A `struct` data type holds field data types and a `sharding_indexed` +codec holds two codec pipelines, and neither can coerce its own +configuration without knowing what names are in scope inside it. + +The shared plumbing lives here too: the member checks every entity needs +and the walk over a configuration that applies them. What stays with the +entity is the table saying which members it has -- that is the part that +is about blosc rather than about entities. +""" + +from __future__ import annotations + +from collections.abc import Mapping as _Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, TypeAlias, TypeVar, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._extension_points import canonical_name + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from typing import Self + + from zarr_metadata.model._validation import ProblemKind + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._extension_points import ExtensionPointField + +EntityT = TypeVar("EntityT", bound="MetadataEntity") + +# A real alias, not a string one: entity modules subscript it as +# `Coerced[Self]` in a return annotation, and not all of them defer +# annotation evaluation. +Coerced: TypeAlias = tuple[EntityT | None, tuple[ValidationProblem, ...]] +"""The entity, or None and every reason the metadata is not one. + +A caller that only wants a verdict reads the problems; one that wants to +go on reading the entity checks for None. Both never happen at once. +""" + +Loc: TypeAlias = "tuple[str | int, ...]" + +TypeCheck: TypeAlias = "Callable[[object, Loc], tuple[ValidationProblem, ...]]" +"""Whether one value has the type a member declares, and where if not.""" + +MemberTypes: TypeAlias = "Mapping[str, tuple[bool, TypeCheck]]" +"""Per configuration member: whether it is required, and its type check.""" + + +def problem( + loc: Loc, message: str, kind: ProblemKind = "invalid_type" +) -> tuple[ValidationProblem, ...]: + """One problem, as the tuple every check returns.""" + return (ValidationProblem(loc, message, kind),) + + +def is_int(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """An integer, and not a bool -- JSON `true` is not the integer 1.""" + if isinstance(value, bool) or not isinstance(value, int): + return problem(loc, f"expected an integer, got {value!r}") + return () + + +def is_str(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, str): + return problem(loc, f"expected a string, got {value!r}") + return () + + +def is_bool(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, bool): + return problem(loc, f"expected a boolean, got {value!r}") + return () + + +def one_of(allowed: tuple[str, ...]) -> TypeCheck: + """A member whose type is a closed set of names.""" + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if value not in allowed: + return problem(loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value") + return () + + return check + + +def sequence_of(element: TypeCheck) -> TypeCheck: + """A member whose type is a sequence, checked element by element.""" + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, (list, tuple)): + return problem(loc, f"expected a sequence, got {value!r}") + elements: tuple[object, ...] = tuple(cast("list[object] | tuple[object, ...]", value)) + return tuple( + found for index, entry in enumerate(elements) for found in element(entry, (*loc, index)) + ) + + return check + + +def coerce_members( + configuration: Mapping[str, object], types: MemberTypes +) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """The members `types` declares, taken from `configuration`. + + Returns what was accepted and every problem found: a missing required + member, a member of the wrong type, and a key the entity does not + declare. Only a key it does not declare is survivable -- a caller can + report it without abandoning the entity -- so problems are returned + rather than raised and the caller decides. + """ + problems: list[ValidationProblem] = [] + members: dict[str, object] = {} + for key in configuration: + if key not in types: + problems.extend(problem(("configuration",), f"unexpected key {key!r}", "unknown_key")) + for key, (required, check) in types.items(): + if key not in configuration: + if required: + problems.extend( + problem(("configuration", key), f"missing required key {key!r}", "missing_key") + ) + continue + found = check(configuration[key], ("configuration", key)) + problems.extend(found) + if len(found) == 0: + members[key] = configuration[key] + return members, tuple(problems) + + +@dataclass(frozen=True, slots=True) +class Context: + """The entities in scope while metadata is being read. + + A scope is not a property of the entities, it is a choice the reader + makes: judging against the specification alone, or against the + specification plus what `zarr-extensions` registers. The two live in + `zarr_metadata.v3._registry`. + """ + + entities: Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]] + + def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: + """The entity `name` denotes at `field`, or None if out of scope. + + Out of scope is not an error: an unknown name may be an extension + this reader does not model, and openness means leaving it unjudged. + """ + return self.entities.get(field, {}).get(canonical_name(field, name)) + + +@dataclass(frozen=True, slots=True) +class MetadataEntity: + """One named entity, coerced from its metadata. + + Subclasses add their configuration members as fields, which is what + makes them well-typed by construction: an instance exists only if + `coerce` accepted the metadata that produced it. + """ + + must_understand: bool = True + + identifier: ClassVar[str] + """The name this entity is registered under. + + Usually the `name` the metadata carries. The raw-bytes data types are + the exception: every `r` spelling is one family, so the family gets + an invented identifier that no real name can collide with. + """ + + member_types: ClassVar[MemberTypes] = {} + """The configuration members, and the type each one takes. + + The same keys as the configuration TypedDict, which is the same as the + constructor signature; `tests/v3/test_entities.py` holds the three + together. + """ + + @classmethod + def coerce(cls, value: object, context: Context) -> Coerced[Self]: + """`value` as this entity, or the reasons it is not one. + + `context` is the scope this reading is happening in; most entities + have no use for it and ignore it. + """ + raise NotImplementedError # pragma: no cover - subclasses implement + + def problems(self) -> tuple[ValidationProblem, ...]: + """Every value of this entity the spec disallows. + + Locations are relative to the entity's `configuration`. Default: + an entity whose type admits only valid values has nothing to add. + """ + return () + + def to_json(self) -> ZarrV3MetadataFieldJSON: + """This entity in its simplest equivalent spelling. + + Subclasses narrow the return type to their own object TypedDict, + which is the JSON form this dataclass models. + """ + raise NotImplementedError # pragma: no cover - subclasses implement + + +def named_configuration( + value: object, +) -> tuple[str | None, Mapping[str, object] | None, bool]: + """Split metadata into `(name, configuration, must_understand)`. + + The shared shape every entity arrives in: a bare name, or an object + carrying one. A `None` name means the value is not a metadata field at + all; a `None` configuration means the bare spelling was used. + """ + if isinstance(value, str): + return value, None, True + if not isinstance(value, _Mapping): + return None, None, True + entry = cast("Mapping[str, object]", value) + name = entry.get("name") + if not isinstance(name, str): + return None, None, True + configuration = entry.get("configuration") + must_understand = entry.get("must_understand", True) + return ( + name, + cast("Mapping[str, object]", configuration) + if isinstance(configuration, _Mapping) + else None, + must_understand if isinstance(must_understand, bool) else True, + ) + + +__all__ = [ + "Coerced", + "Context", + "Loc", + "MemberTypes", + "MetadataEntity", + "TypeCheck", + "coerce_members", + "is_bool", + "is_int", + "is_str", + "named_configuration", + "one_of", + "problem", + "sequence_of", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py new file mode 100644 index 0000000000..8cd327306e --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -0,0 +1,75 @@ +"""Which entities are in scope when raw JSON is coerced. + +The only registry the package needs. A validator reading a document has +to decide, for each extension point, which identifier maps to which +entity — and that decision is the *scope* it validates against, not a +property of the entities themselves. + +Two scopes, because the question "is this document valid?" has two useful +answers. `CORE` is what the Zarr v3 specification itself defines, so a +document validating against it uses nothing an implementation could +refuse for being optional. `CORE_AND_EXTENSIONS` adds what +`zarr-extensions` registers and this package models. A name in neither is +not rejected — extension openness — it is simply not judged. + +Identifiers are the `name` the metadata carries, with one exception. Every +`r` spelling is one data-type family, so the family registers under an +invented identifier that no real name can collide with; `canonical_name` +folds a spelling onto it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from zarr_metadata.v3._entity import Context +from zarr_metadata.v3._extension_points import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, +) +from zarr_metadata.v3.codec.blosc import BloscCodec + +if TYPE_CHECKING: + from zarr_metadata.v3._entity import MetadataEntity + +_CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { + BloscCodec.identifier: BloscCodec, +} +_EXTENSION_CODECS: Final[dict[str, type[MetadataEntity]]] = {} + +_CORE_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} +_EXTENSION_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} + +_CORE_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = {} +_EXTENSION_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = {} + +_CORE_CHUNK_KEY_ENCODINGS: Final[dict[str, type[MetadataEntity]]] = {} + + +CORE: Final = Context( + { + CODECS: _CORE_CODECS, + DATA_TYPE: _CORE_DATA_TYPES, + CHUNK_GRID: _CORE_CHUNK_GRIDS, + CHUNK_KEY_ENCODING: _CORE_CHUNK_KEY_ENCODINGS, + } +) +"""Only what the Zarr v3 specification defines.""" + +CORE_AND_EXTENSIONS: Final = Context( + { + CODECS: {**_CORE_CODECS, **_EXTENSION_CODECS}, + DATA_TYPE: {**_CORE_DATA_TYPES, **_EXTENSION_DATA_TYPES}, + CHUNK_GRID: {**_CORE_CHUNK_GRIDS, **_EXTENSION_CHUNK_GRIDS}, + CHUNK_KEY_ENCODING: _CORE_CHUNK_KEY_ENCODINGS, + } +) +"""What the specification defines, plus what `zarr-extensions` registers.""" + + +__all__ = [ + "CORE", + "CORE_AND_EXTENSIONS", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 5cce93fdbf..ab357ed1c2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,9 +5,23 @@ """ from collections.abc import Mapping -from typing import Final, Literal, NotRequired - -from typing_extensions import TypedDict +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, Self, cast + +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + Coerced, + Context, + MemberTypes, + MetadataEntity, + coerce_members, + is_int, + named_configuration, + one_of, + problem, +) BLOSC_CODEC_NAME: Final = "blosc" """The `name` field value of the `blosc` codec.""" @@ -93,3 +107,114 @@ def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, "BloscShuffle", "canonical_configuration", ] + + +@dataclass(frozen=True, slots=True) +class BloscCodec(MetadataEntity): + """The `blosc` codec, coerced from its metadata. + + Everything blosc knows about itself: the shape its metadata takes, the + values the spec allows in it, and the simplest spelling of an + equivalent document. + """ + + cname: BloscCName = "zstd" + clevel: int = 5 + shuffle: BloscShuffle = "noshuffle" + blocksize: int = 0 + typesize: int | None = None + + identifier: ClassVar[str] = BLOSC_CODEC_NAME + kind: ClassVar[str] = "bytes_bytes" + + # Every member is required but `typesize`, which only means something + # when shuffling; `problems` is where that conditional lives. + member_types: ClassVar[MemberTypes] = { + "cname": (True, one_of(BLOSC_CNAME)), + "clevel": (True, is_int), + "shuffle": (True, one_of(BLOSC_SHUFFLE)), + "blocksize": (True, is_int), + "typesize": (False, is_int), + } + + @classmethod + def coerce(cls, value: object, context: Context) -> Coerced[Self]: + name, configuration, must_understand = named_configuration(value) + if name != BLOSC_CODEC_NAME: + return None, problem((), f"expected the {BLOSC_CODEC_NAME!r} codec") + if configuration is None: + # Required members mean the bare-name spelling says too little. + return None, problem( + ("configuration",), "blosc requires a configuration", "missing_key" + ) + members, found = coerce_members(configuration, cls.member_types) + if any(entry.kind != "unknown_key" for entry in found): + return None, found + return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] + + def problems(self) -> tuple[ValidationProblem, ...]: + """The value constraints the spec places on a blosc configuration.""" + found: list[ValidationProblem] = [] + if not 0 <= self.clevel <= 9: + found.extend( + problem( + ("clevel",), + f"expected an integer in [0, 9], got {self.clevel}", + "invalid_value", + ) + ) + if self.blocksize < 0: + found.extend( + problem( + ("blocksize",), + f"expected a non-negative integer, got {self.blocksize}", + "invalid_value", + ) + ) + if self.typesize is not None and self.typesize < 1: + found.extend( + problem( + ("typesize",), + f"expected a positive integer, got {self.typesize}", + "invalid_value", + ) + ) + if self.shuffle != BLOSC_NO_SHUFFLE and self.typesize is None: + found.extend( + problem( + ("typesize",), + f"typesize is required when shuffle is {self.shuffle!r}", + "missing_key", + ) + ) + return tuple(found) + + @classmethod + def from_configuration(cls, **configuration: Unpack[BloscCodecConfiguration]) -> Self: + """This codec from its configuration members. + + The configuration TypedDict unpacked *is* this constructor's + signature, so a caller with a well-typed configuration builds a + well-typed codec, and a type checker says so at the call site. + """ + return cls(**configuration) + + def to_json(self) -> BloscCodecObject: + """The simplest spelling of this codec. + + `typesize` is dropped under `noshuffle`, where the spec says of it + that "the value is ignored" — so two documents differing only there + describe the same codec. + """ + configuration: dict[str, object] = { + "cname": self.cname, + "clevel": self.clevel, + "shuffle": self.shuffle, + "blocksize": self.blocksize, + } + if self.typesize is not None and self.shuffle != BLOSC_NO_SHUFFLE: + configuration["typesize"] = self.typesize + entry: dict[str, object] = {"name": BLOSC_CODEC_NAME, "configuration": configuration} + if not self.must_understand: + entry["must_understand"] = False + return cast("BloscCodecObject", entry) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py new file mode 100644 index 0000000000..7ed7b90af3 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -0,0 +1,73 @@ +"""The correspondence between an entity's dataclass and its TypedDicts. + +A configuration TypedDict unpacked is exactly the dataclass constructor's +signature, and the object TypedDict is exactly what `to_json` returns. +Asserting the first is what lets the dataclass carry its own checks +instead of a hand-written per-member table elsewhere: the two cannot +drift, because one drifting makes this fail. +""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING, get_type_hints + +import pytest + +from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS +from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration + +if TYPE_CHECKING: + from zarr_metadata.v3._entity import MetadataEntity + +# Each registered entity, paired with the TypedDict its constructor mirrors. +CONFIGURATIONS: dict[str, tuple[type[MetadataEntity], type]] = { + "blosc": (BloscCodec, BloscCodecConfiguration), +} + + +@pytest.mark.parametrize( + ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) +) +def test_the_constructor_mirrors_the_configuration( + entity: type[MetadataEntity], configuration: type +) -> None: + # `must_understand` belongs to the object, not the configuration, so it + # is the one field the two deliberately do not share. + fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} + assert fields == set(get_type_hints(configuration)) + + +def test_every_registered_entity_is_checked_here() -> None: + registered = { + identifier for entities in CORE_AND_EXTENSIONS.entities.values() for identifier in entities + } + assert registered == set(CONFIGURATIONS) + + +def test_core_is_a_subset_of_core_and_extensions() -> None: + for field, entities in CORE.entities.items(): + assert entities.items() <= CORE_AND_EXTENSIONS.entities[field].items() + + +def test_a_name_out_of_scope_resolves_to_nothing() -> None: + # Not an error: an unmodelled extension is left unjudged, not rejected. + assert CORE.resolve("codecs", "mycorp.secret") is None + assert CORE.resolve("codecs", "blosc") is BloscCodec + + +def test_an_entity_round_trips_through_its_json_form() -> None: + original = { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "blocksize": 0, + "typesize": 4, + }, + } + codec, problems = BloscCodec.coerce(original, CORE) + assert problems == () + assert codec is not None + assert codec.to_json() == original From bafcb8f0fdcce91954a1f7b43dff11babc8b7494 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 16:52:01 +0200 Subject: [PATCH 025/107] feat(zarr-metadata): the rest of the simple codecs, as entities `coerce` and `to_json` turned out to be the same for every entity that does not contain another one: check the name, walk the member table, construct. So they moved into the base, driven by two class variables, and a codec now declares what is particular to it -- its members, its pipeline kind, the values the spec disallows -- and nothing else. Three spellings of the same set of members now exist: the dataclass fields, the configuration TypedDict, and the member table. Two new tests hold all three together, and a third derives `configuration_required` from the TypedDict's required keys, because the spec ties the bare-name spelling to exactly that. `CodecKind` moved next to the entity base. It had to: `codec.kind` imports every codec module to build its tuples, so no codec module could import back from it. No slots on entities. `slots=True` rebuilds the class and leaves a subclass's zero-argument `super()` pointing at the class it replaced, and every entity calls `super()` to narrow `to_json` to its own object TypedDict. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 116 ++++++++++++++++-- .../src/zarr_metadata/v3/_registry.py | 15 ++- .../src/zarr_metadata/v3/codec/blosc.py | 51 +++----- .../src/zarr_metadata/v3/codec/bytes.py | 30 ++++- .../src/zarr_metadata/v3/codec/crc32c.py | 23 +++- .../src/zarr_metadata/v3/codec/gzip.py | 37 +++++- .../src/zarr_metadata/v3/codec/kind.py | 6 +- .../zarr_metadata/v3/codec/scale_offset.py | 34 ++++- .../src/zarr_metadata/v3/codec/transpose.py | 44 ++++++- .../src/zarr_metadata/v3/codec/zstd.py | 52 +++++++- .../zarr-metadata/tests/test_public_api.py | 7 ++ .../zarr-metadata/tests/v3/test_entities.py | 36 ++++++ 12 files changed, 397 insertions(+), 54 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 30f5b74b4d..9dd07df877 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -40,9 +40,9 @@ from collections.abc import Mapping as _Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, ClassVar, Literal, TypeAlias, TypeVar, cast -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import ValidationProblem, is_json from zarr_metadata.v3._extension_points import canonical_name if TYPE_CHECKING: @@ -67,6 +67,14 @@ Loc: TypeAlias = "tuple[str | int, ...]" +CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] +"""The three pipeline positions the v3 spec sorts codecs into. + +Here rather than in `zarr_metadata.v3.codec.kind` because each codec +declares its own kind, and that module imports every codec to build the +tuples it will no longer need once they all do. +""" + TypeCheck: TypeAlias = "Callable[[object, Loc], tuple[ValidationProblem, ...]]" """Whether one value has the type a member declares, and where if not.""" @@ -100,6 +108,13 @@ def is_bool(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return () +def is_json_value(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """Any JSON value at all -- the widest type a member can declare.""" + if not is_json(value): + return problem(loc, f"expected a JSON value, got {value!r}") + return () + + def one_of(allowed: tuple[str, ...]) -> TypeCheck: """A member whose type is a closed set of names.""" @@ -125,6 +140,22 @@ def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return check +def _as_tuples(value: object) -> object: + """Every JSON array in `value`, at any depth, as a tuple. + + The TypedDicts spell a JSON array as a tuple throughout, so a member + taken straight from parsed JSON would otherwise hold a list where its + own type says tuple -- and two documents differing only in that would + compare unequal. + """ + if isinstance(value, list): + return tuple(_as_tuples(entry) for entry in cast("list[object]", value)) + if isinstance(value, _Mapping): + entries = cast("Mapping[str, object]", value) + return {key: _as_tuples(entry) for key, entry in entries.items()} + return value + + def coerce_members( configuration: Mapping[str, object], types: MemberTypes ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: @@ -151,7 +182,7 @@ def coerce_members( found = check(configuration[key], ("configuration", key)) problems.extend(found) if len(found) == 0: - members[key] = configuration[key] + members[key] = _as_tuples(configuration[key]) return members, tuple(problems) @@ -176,13 +207,25 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] return self.entities.get(field, {}).get(canonical_name(field, name)) -@dataclass(frozen=True, slots=True) +# No `slots=True`, deliberately: it rebuilds the class, which leaves the +# zero-argument `super()` in a subclass pointing at the class that was +# replaced. Subclasses call `super()` to narrow `to_json` and to adjust +# `configuration`, so slots would be a trap laid for every entity. +@dataclass(frozen=True) class MetadataEntity: """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what makes them well-typed by construction: an instance exists only if - `coerce` accepted the metadata that produced it. + `coerce` accepted the metadata that produced it. An optional member is + typed `| None` with a default of `None`, so absence is representable + and a canonical spelling can leave it out. + + Most subclasses declare `member_types` and nothing else: the default + `coerce` and `to_json` are written once here against that table. The + ones that override are the ones with something particular to say -- + a configuration containing other entities, a name that is a family + rather than a constant, a member another member renders meaningless. """ must_understand: bool = True @@ -203,6 +246,22 @@ class MetadataEntity: together. """ + configuration_required: ClassVar[bool] = False + """Whether the bare-name spelling says too little for this entity. + + The spec permits a bare name "if no configuration metadata is + required", so this is true exactly when some member is required. + """ + + @classmethod + def accepts(cls, name: str) -> bool: + """Whether `name` denotes this entity. + + Constant for all but the raw-bytes family, where one class covers + every `r`. + """ + return name == cls.identifier + @classmethod def coerce(cls, value: object, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one. @@ -210,7 +269,36 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: `context` is the scope this reading is happening in; most entities have no use for it and ignore it. """ - raise NotImplementedError # pragma: no cover - subclasses implement + name, configuration, must_understand = named_configuration(value) + if name is None or not cls.accepts(name): + return None, problem((), f"expected the {cls.identifier!r} entity") + if configuration is None: + if cls.configuration_required: + return None, problem( + ("configuration",), + f"{cls.identifier!r} requires a configuration", + "missing_key", + ) + configuration = cast("Mapping[str, object]", {}) + members, found = coerce_members(configuration, cls.member_types) + # An unknown key is worth reporting but does not stop the entity + # from being read: every member it declares was still understood. + if any(entry.kind != "unknown_key" for entry in found): + return None, found + return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] + + def configuration(self) -> dict[str, object]: + """This entity's configuration, in its simplest equivalent form. + + Absent optional members are left out, which is what makes the + bare-name spelling reachable. Override to drop a member that + another member renders meaningless. + """ + return { + key: value + for key in type(self).member_types + if (value := getattr(self, key)) is not None + } def problems(self) -> tuple[ValidationProblem, ...]: """Every value of this entity the spec disallows. @@ -223,10 +311,22 @@ def problems(self) -> tuple[ValidationProblem, ...]: def to_json(self) -> ZarrV3MetadataFieldJSON: """This entity in its simplest equivalent spelling. + A name alone when the name says everything, and the object form + otherwise. `must_understand` is omitted when true, because that is + the default and says nothing; an explicit false says something. + Subclasses narrow the return type to their own object TypedDict, which is the JSON form this dataclass models. """ - raise NotImplementedError # pragma: no cover - subclasses implement + configuration = self.configuration() + if len(configuration) == 0 and self.must_understand: + return cast("ZarrV3MetadataFieldJSON", type(self).identifier) + entry: dict[str, object] = {"name": type(self).identifier} + if len(configuration) != 0: + entry["configuration"] = configuration + if not self.must_understand: + entry["must_understand"] = False + return cast("ZarrV3MetadataFieldJSON", entry) def named_configuration( @@ -258,6 +358,7 @@ def named_configuration( __all__ = [ + "CodecKind", "Coerced", "Context", "Loc", @@ -267,6 +368,7 @@ def named_configuration( "coerce_members", "is_bool", "is_int", + "is_json_value", "is_str", "named_configuration", "one_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 8cd327306e..ad98fa7b13 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -30,14 +30,27 @@ DATA_TYPE, ) from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.codec.bytes import BytesCodec +from zarr_metadata.v3.codec.crc32c import Crc32cCodec +from zarr_metadata.v3.codec.gzip import GzipCodec +from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec +from zarr_metadata.v3.codec.transpose import TransposeCodec +from zarr_metadata.v3.codec.zstd import ZstdCodec if TYPE_CHECKING: from zarr_metadata.v3._entity import MetadataEntity _CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { BloscCodec.identifier: BloscCodec, + BytesCodec.identifier: BytesCodec, + Crc32cCodec.identifier: Crc32cCodec, + GzipCodec.identifier: GzipCodec, + TransposeCodec.identifier: TransposeCodec, +} +_EXTENSION_CODECS: Final[dict[str, type[MetadataEntity]]] = { + ScaleOffsetCodec.identifier: ScaleOffsetCodec, + ZstdCodec.identifier: ZstdCodec, } -_EXTENSION_CODECS: Final[dict[str, type[MetadataEntity]]] = {} _CORE_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} _EXTENSION_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index ab357ed1c2..a6b657ff9e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -12,13 +12,9 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( - Coerced, - Context, MemberTypes, MetadataEntity, - coerce_members, is_int, - named_configuration, one_of, problem, ) @@ -100,6 +96,7 @@ def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, "BLOSC_NO_SHUFFLE", "BLOSC_SHUFFLE", "BloscCName", + "BloscCodec", "BloscCodecConfiguration", "BloscCodecMetadata", "BloscCodecName", @@ -109,7 +106,7 @@ def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, ] -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True) class BloscCodec(MetadataEntity): """The `blosc` codec, coerced from its metadata. @@ -129,6 +126,8 @@ class BloscCodec(MetadataEntity): # Every member is required but `typesize`, which only means something # when shuffling; `problems` is where that conditional lives. + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { "cname": (True, one_of(BLOSC_CNAME)), "clevel": (True, is_int), @@ -137,21 +136,6 @@ class BloscCodec(MetadataEntity): "typesize": (False, is_int), } - @classmethod - def coerce(cls, value: object, context: Context) -> Coerced[Self]: - name, configuration, must_understand = named_configuration(value) - if name != BLOSC_CODEC_NAME: - return None, problem((), f"expected the {BLOSC_CODEC_NAME!r} codec") - if configuration is None: - # Required members mean the bare-name spelling says too little. - return None, problem( - ("configuration",), "blosc requires a configuration", "missing_key" - ) - members, found = coerce_members(configuration, cls.member_types) - if any(entry.kind != "unknown_key" for entry in found): - return None, found - return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] - def problems(self) -> tuple[ValidationProblem, ...]: """The value constraints the spec places on a blosc configuration.""" found: list[ValidationProblem] = [] @@ -199,22 +183,17 @@ def from_configuration(cls, **configuration: Unpack[BloscCodecConfiguration]) -> """ return cls(**configuration) - def to_json(self) -> BloscCodecObject: - """The simplest spelling of this codec. + def configuration(self) -> dict[str, object]: + """The simplest spelling of this codec's configuration. `typesize` is dropped under `noshuffle`, where the spec says of it - that "the value is ignored" — so two documents differing only there - describe the same codec. + that "the value is ignored" -- so two documents differing only + there describe the same codec. """ - configuration: dict[str, object] = { - "cname": self.cname, - "clevel": self.clevel, - "shuffle": self.shuffle, - "blocksize": self.blocksize, - } - if self.typesize is not None and self.shuffle != BLOSC_NO_SHUFFLE: - configuration["typesize"] = self.typesize - entry: dict[str, object] = {"name": BLOSC_CODEC_NAME, "configuration": configuration} - if not self.must_understand: - entry["must_understand"] = False - return cast("BloscCodecObject", entry) + members = super().configuration() + if self.shuffle == BLOSC_NO_SHUFFLE: + members.pop("typesize", None) + return members + + def to_json(self) -> BloscCodecObject: + return cast("BloscCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 4feb6b8c1c..f06c434d24 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -4,10 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/bytes/index.html """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.v3._entity import ( + CodecKind, + MemberTypes, + MetadataEntity, + one_of, +) + BYTES_CODEC_NAME: Final = "bytes" """The `name` field value of the `bytes` codec.""" @@ -59,9 +67,29 @@ class BytesCodecObject(TypedDict, closed=True): __all__ = [ "BYTES_CODEC_NAME", "ENDIANNESS", + "BytesCodec", "BytesCodecConfiguration", "BytesCodecMetadata", "BytesCodecName", "BytesCodecObject", "Endianness", ] + + +@dataclass(frozen=True) +class BytesCodec(MetadataEntity): + """The `bytes` codec, coerced from its metadata. + + `endian` is optional and absent means something: a one-byte data type + has no byte order to state, and the spec lets such an array omit it. + """ + + endian: Endianness | None = None + + identifier: ClassVar[str] = BYTES_CODEC_NAME + kind: ClassVar[CodecKind] = "array_bytes" + + member_types: ClassVar[MemberTypes] = {"endian": (False, one_of(ENDIANNESS))} + + def to_json(self) -> BytesCodecObject | BytesCodecName: + return cast("BytesCodecObject | BytesCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 05661d0b59..37cd77157c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -7,10 +7,16 @@ key is absent from the metadata. """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.v3._entity import ( + CodecKind, + MetadataEntity, +) + CRC32C_CODEC_NAME: Final = "crc32c" """The `name` field value of the `crc32c` codec.""" @@ -47,7 +53,22 @@ class Crc32cCodecObject(TypedDict, closed=True): __all__ = [ "CRC32C_CODEC_NAME", + "Crc32cCodec", "Crc32cCodecMetadata", "Crc32cCodecName", "Crc32cCodecObject", ] + + +@dataclass(frozen=True) +class Crc32cCodec(MetadataEntity): + """The `crc32c` codec, coerced from its metadata. + + The name says everything: a checksum has nothing to configure. + """ + + identifier: ClassVar[str] = CRC32C_CODEC_NAME + kind: ClassVar[CodecKind] = "bytes_bytes" + + def to_json(self) -> Crc32cCodecObject | Crc32cCodecName: + return cast("Crc32cCodecObject | Crc32cCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index d516088f99..0d602d2e04 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -4,10 +4,20 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + CodecKind, + MemberTypes, + MetadataEntity, + is_int, + problem, +) + GZIP_CODEC_NAME: Final = "gzip" """The `name` field value of the `gzip` codec.""" @@ -50,8 +60,33 @@ class GzipCodecObject(TypedDict, closed=True): __all__ = [ "GZIP_CODEC_NAME", + "GzipCodec", "GzipCodecConfiguration", "GzipCodecMetadata", "GzipCodecName", "GzipCodecObject", ] + + +@dataclass(frozen=True) +class GzipCodec(MetadataEntity): + """The `gzip` codec, coerced from its metadata.""" + + level: int = 5 + + identifier: ClassVar[str] = GZIP_CODEC_NAME + kind: ClassVar[CodecKind] = "bytes_bytes" + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = {"level": (True, is_int)} + + def problems(self) -> tuple[ValidationProblem, ...]: + """gzip compression levels run 0 to 9.""" + if not 0 <= self.level <= 9: + return problem( + ("level",), f"expected an integer in [0, 9], got {self.level}", "invalid_value" + ) + return () + + def to_json(self) -> GzipCodecObject: + return cast("GzipCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py index a5853c0f97..6578719cea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py @@ -8,8 +8,9 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ -from typing import Final, Literal +from typing import Final +from zarr_metadata.v3._entity import CodecKind from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME @@ -38,9 +39,6 @@ ) """Tuple of the `name` field values of the known `bytes -> bytes` codecs.""" -CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] -"""The three pipeline positions the v3 spec sorts codecs into.""" - def codec_kind_of_name(name: str) -> CodecKind | None: """The pipeline kind of the codec named `name`, or None if unknown. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 344a4c435c..ea5b12a2f8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -4,11 +4,18 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict from zarr_metadata._common import JSONValue +from zarr_metadata.v3._entity import ( + CodecKind, + MemberTypes, + MetadataEntity, + is_json_value, +) SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" """The `name` field value of the `scale_offset` codec.""" @@ -56,8 +63,33 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): __all__ = [ "SCALE_OFFSET_CODEC_NAME", + "ScaleOffsetCodec", "ScaleOffsetCodecConfiguration", "ScaleOffsetCodecMetadata", "ScaleOffsetCodecName", "ScaleOffsetCodecObject", ] + + +@dataclass(frozen=True) +class ScaleOffsetCodec(MetadataEntity): + """The `scale_offset` codec, coerced from its metadata. + + Both members are optional and any JSON scalar is well-typed here; what + a given value means depends on the data type it is applied to, which + is a question for the rules layer. + """ + + offset: JSONValue | None = None + scale: JSONValue | None = None + + identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME + kind: ClassVar[CodecKind] = "array_array" + + member_types: ClassVar[MemberTypes] = { + "offset": (False, is_json_value), + "scale": (False, is_json_value), + } + + def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: + return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 8024174605..2deccbfa4e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -4,10 +4,21 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + CodecKind, + MemberTypes, + MetadataEntity, + is_int, + problem, + sequence_of, +) + TRANSPOSE_CODEC_NAME: Final = "transpose" """The `name` field value of the `transpose` codec.""" @@ -45,8 +56,39 @@ class TransposeCodecObject(TypedDict, closed=True): __all__ = [ "TRANSPOSE_CODEC_NAME", + "TransposeCodec", "TransposeCodecConfiguration", "TransposeCodecMetadata", "TransposeCodecName", "TransposeCodecObject", ] + + +@dataclass(frozen=True) +class TransposeCodec(MetadataEntity): + """The `transpose` codec, coerced from its metadata.""" + + order: tuple[int, ...] = () + + identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME + kind: ClassVar[CodecKind] = "array_array" + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = {"order": (True, sequence_of(is_int))} + + def problems(self) -> tuple[ValidationProblem, ...]: + """`order` must permute its own axes. + + Whether it permutes the *array's* axes is a different question -- + it needs the array's rank -- and the rules layer asks that one. + """ + if sorted(self.order) != list(range(len(self.order))): + return problem( + ("order",), + f"expected a permutation of 0..{len(self.order) - 1}, got {self.order!r}", + "invalid_value", + ) + return () + + def to_json(self) -> TransposeCodecObject: + return cast("TransposeCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index b7ee3f5685..7db38b8842 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -6,16 +6,33 @@ proposed the codec, was never merged). """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + CodecKind, + MemberTypes, + MetadataEntity, + is_bool, + is_int, + problem, +) + ZSTD_CODEC_NAME: Final = "zstd" """The `name` field value of the `zstd` codec.""" ZstdCodecName = Literal["zstd"] """Literal type of the `name` field of the `zstd` codec.""" +ZSTD_MIN_LEVEL: Final = -131072 +"""The lowest `level` zstd accepts: ZSTD_minCLevel(), -(1 << 17).""" + +ZSTD_MAX_LEVEL: Final = 22 +"""The highest `level` zstd accepts: ZSTD_maxCLevel().""" + class ZstdCodecConfiguration(TypedDict, closed=True): """ @@ -49,8 +66,41 @@ class ZstdCodecObject(TypedDict, closed=True): __all__ = [ "ZSTD_CODEC_NAME", + "ZSTD_MAX_LEVEL", + "ZSTD_MIN_LEVEL", + "ZstdCodec", "ZstdCodecConfiguration", "ZstdCodecMetadata", "ZstdCodecName", "ZstdCodecObject", ] + + +@dataclass(frozen=True) +class ZstdCodec(MetadataEntity): + """The `zstd` codec, coerced from its metadata.""" + + level: int = 0 + checksum: bool | None = None + + identifier: ClassVar[str] = ZSTD_CODEC_NAME + kind: ClassVar[CodecKind] = "bytes_bytes" + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { + "level": (True, is_int), + "checksum": (False, is_bool), + } + + def problems(self) -> tuple[ValidationProblem, ...]: + """zstd compression levels run -131072 to 22.""" + if not ZSTD_MIN_LEVEL <= self.level <= ZSTD_MAX_LEVEL: + return problem( + ("level",), + f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {self.level}", + "invalid_value", + ) + return () + + def to_json(self) -> ZstdCodecObject: + return cast("ZstdCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index fafa67ff59..55766080da 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -267,6 +267,13 @@ def test_all_is_grouped_and_unique() -> None: "FillValue", "Configuration", "Component", + # The bare role is the coerced entity itself: the dataclass that owns + # the extension's type checks, value checks and canonical spelling. + # Listed last so a longer role still wins the alternation. + "Codec", + "ChunkGrid", + "ChunkKeyEncoding", + "DataType", ) _EXTENSION_NAME = re.compile(r"^(?:[A-Z][a-z0-9]*)+?(?:" + "|".join(_EXTENSION_ROLES) + r")$") diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 7ed7b90af3..95d0cec400 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -16,6 +16,12 @@ from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration +from zarr_metadata.v3.codec.bytes import BytesCodec, BytesCodecConfiguration +from zarr_metadata.v3.codec.crc32c import Crc32cCodec, Empty +from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecConfiguration +from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec, ScaleOffsetCodecConfiguration +from zarr_metadata.v3.codec.transpose import TransposeCodec, TransposeCodecConfiguration +from zarr_metadata.v3.codec.zstd import ZstdCodec, ZstdCodecConfiguration if TYPE_CHECKING: from zarr_metadata.v3._entity import MetadataEntity @@ -23,6 +29,12 @@ # Each registered entity, paired with the TypedDict its constructor mirrors. CONFIGURATIONS: dict[str, tuple[type[MetadataEntity], type]] = { "blosc": (BloscCodec, BloscCodecConfiguration), + "bytes": (BytesCodec, BytesCodecConfiguration), + "crc32c": (Crc32cCodec, Empty), + "gzip": (GzipCodec, GzipCodecConfiguration), + "scale_offset": (ScaleOffsetCodec, ScaleOffsetCodecConfiguration), + "transpose": (TransposeCodec, TransposeCodecConfiguration), + "zstd": (ZstdCodec, ZstdCodecConfiguration), } @@ -38,6 +50,30 @@ def test_the_constructor_mirrors_the_configuration( assert fields == set(get_type_hints(configuration)) +@pytest.mark.parametrize( + ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) +) +def test_the_member_table_mirrors_the_configuration( + entity: type[MetadataEntity], configuration: type +) -> None: + # The third spelling of the same set. Which members are *required* is + # in the TypedDict too, so that cannot drift either. + assert set(entity.member_types) == set(get_type_hints(configuration)) + required = {key for key, (needed, _) in entity.member_types.items() if needed} + assert required == set(configuration.__required_keys__) # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) +) +def test_a_required_member_rules_out_the_bare_spelling( + entity: type[MetadataEntity], configuration: type +) -> None: + # The spec permits a bare name only "if no configuration metadata is + # required", so one flag follows from the other. + assert entity.configuration_required == (len(configuration.__required_keys__) != 0) # type: ignore[attr-defined] + + def test_every_registered_entity_is_checked_here() -> None: registered = { identifier for entities in CORE_AND_EXTENSIONS.entities.values() for identifier in entities From 017126c23417fd3cebc189a56343fd93e377e93d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 16:54:32 +0200 Subject: [PATCH 026/107] feat(zarr-metadata): the chunk grids and chunk key encodings, as entities Chunk-extent positivity moves onto the grid that declares the extents, where it can point at the offending axis. What needs the array's shape -- one extent per dimension, explicit specs summing to the extent -- stays in the rules layer, because a grid cannot answer it alone. Rectilinear's run-length encoding becomes its canonical `configuration`, and its dimension specs get a type check written out rather than composed from `sequence_of`: an entry is an extent or a `[size, count]` run, which `sequence_of` cannot say. Coercion normalizes JSON arrays to tuples at every depth before checking them, so a member never holds a list where its own type says tuple. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 21 ++- .../src/zarr_metadata/v3/_registry.py | 17 ++- .../v3/chunk_grid/rectilinear.py | 124 +++++++++++++++++- .../zarr_metadata/v3/chunk_grid/regular.py | 45 ++++++- .../v3/chunk_key_encoding/default.py | 28 +++- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 26 +++- .../zarr-metadata/tests/v3/test_entities.py | 17 +++ 7 files changed, 268 insertions(+), 10 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 9dd07df877..3982a7535a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -42,6 +42,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Literal, TypeAlias, TypeVar, cast +from typing_extensions import TypeIs + from zarr_metadata.model._validation import ValidationProblem, is_json from zarr_metadata.v3._extension_points import canonical_name @@ -89,9 +91,18 @@ def problem( return (ValidationProblem(loc, message, kind),) +def is_integer(value: object) -> TypeIs[int]: + """A JSON integer: an `int`, and not a `bool`. + + `True` is an `int` in Python and `true` is not a number in JSON, so + the two have to be told apart everywhere a number is expected. + """ + return not isinstance(value, bool) and isinstance(value, int) + + def is_int(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: """An integer, and not a bool -- JSON `true` is not the integer 1.""" - if isinstance(value, bool) or not isinstance(value, int): + if not is_integer(value): return problem(loc, f"expected an integer, got {value!r}") return () @@ -179,10 +190,13 @@ def coerce_members( problem(("configuration", key), f"missing required key {key!r}", "missing_key") ) continue - found = check(configuration[key], ("configuration", key)) + # Normalized before the check, so a check only ever sees the tuples + # the TypedDicts declare -- never the lists raw JSON arrives as. + value = _as_tuples(configuration[key]) + found = check(value, ("configuration", key)) problems.extend(found) if len(found) == 0: - members[key] = _as_tuples(configuration[key]) + members[key] = value return members, tuple(problems) @@ -368,6 +382,7 @@ def named_configuration( "coerce_members", "is_bool", "is_int", + "is_integer", "is_json_value", "is_str", "named_configuration", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index ad98fa7b13..3d763f21cc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -29,6 +29,10 @@ CODECS, DATA_TYPE, ) +from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid +from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid +from zarr_metadata.v3.chunk_key_encoding.default import DefaultChunkKeyEncoding +from zarr_metadata.v3.chunk_key_encoding.v2 import V2ChunkKeyEncoding from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.bytes import BytesCodec from zarr_metadata.v3.codec.crc32c import Crc32cCodec @@ -55,10 +59,17 @@ _CORE_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} _EXTENSION_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} -_CORE_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = {} -_EXTENSION_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = {} +_CORE_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = { + RegularChunkGrid.identifier: RegularChunkGrid, +} +_EXTENSION_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = { + RectilinearChunkGrid.identifier: RectilinearChunkGrid, +} -_CORE_CHUNK_KEY_ENCODINGS: Final[dict[str, type[MetadataEntity]]] = {} +_CORE_CHUNK_KEY_ENCODINGS: Final[dict[str, type[MetadataEntity]]] = { + DefaultChunkKeyEncoding.identifier: DefaultChunkKeyEncoding, + V2ChunkKeyEncoding.identifier: V2ChunkKeyEncoding, +} CORE: Final = Context( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index be2332fa86..bfa6f30617 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -4,13 +4,31 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + Loc, + MemberTypes, + MetadataEntity, + is_integer, + one_of, + problem, +) + RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" """The `name` field value of the rectilinear chunk grid.""" +RECTILINEAR_CHUNK_GRID_KIND: Final = ("inline",) +"""The `kind` values the rectilinear grid defines. + +Only `inline` so far: the extents are written into the metadata. The +member exists so a later kind can put them somewhere else. +""" + RectilinearChunkGridName = Literal["rectilinear"] """Literal type of the `name` field of the rectilinear chunk grid.""" @@ -89,6 +107,7 @@ def canonical_chunk_shapes( __all__ = [ "RECTILINEAR_CHUNK_GRID_NAME", + "RectilinearChunkGrid", "RectilinearChunkGridConfiguration", "RectilinearChunkGridMetadata", "RectilinearChunkGridName", @@ -97,3 +116,106 @@ def canonical_chunk_shapes( "canonical_chunk_shapes", "canonical_dim_spec", ] + + +def _is_dim_specs(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """One spec per dimension, each a bare extent or a list of entries. + + An entry is an extent or a `[size, count]` run. The nesting is why + this is written out rather than composed from `sequence_of`. + """ + if not isinstance(value, tuple): + return problem(loc, f"expected an array of dimension specs, got {value!r}") + specs = cast("tuple[object, ...]", value) + found: list[ValidationProblem] = [] + for dim, spec in enumerate(specs): + at: Loc = (*loc, dim) + if is_integer(spec): + continue + if not isinstance(spec, tuple): + found.extend( + problem( + at, + "expected an integer or an array of integers / [value, count] pairs, " + f"got {spec!r}", + ) + ) + continue + for position, item in enumerate(cast("tuple[object, ...]", spec)): + if is_integer(item): + continue + entries = cast("tuple[object, ...]", item) if isinstance(item, tuple) else () + if len(entries) == 2 and all(is_integer(part) for part in entries): + continue + found.extend( + problem( + (*at, position), f"expected an integer or a [value, count] pair, got {item!r}" + ) + ) + return tuple(found) + + +@dataclass(frozen=True) +class RectilinearChunkGrid(MetadataEntity): + """The `rectilinear` chunk grid, coerced from its metadata.""" + + kind: Literal["inline"] = "inline" + chunk_shapes: tuple[RectilinearDimSpec, ...] = () + + identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { + "kind": (True, one_of(RECTILINEAR_CHUNK_GRID_KIND)), + "chunk_shapes": (True, _is_dim_specs), + } + + def problems(self) -> tuple[ValidationProblem, ...]: + """Every chunk extent, bare or run-length encoded, must be positive. + + A run's count must be positive too: a run of zero chunks is a way + of writing nothing at all, and the empty spelling already exists. + """ + found: list[ValidationProblem] = [] + for dim, spec in enumerate(self.chunk_shapes): + loc: tuple[str | int, ...] = ("chunk_shapes", dim) + if isinstance(spec, int): + if spec < 1: + found.extend( + problem( + loc, f"expected a positive chunk extent, got {spec}", "invalid_value" + ) + ) + continue + for position, item in enumerate(spec): + if isinstance(item, int): + if item < 1: + found.extend( + problem( + (*loc, position), + f"expected a positive chunk extent, got {item}", + "invalid_value", + ) + ) + elif item[0] < 1 or item[1] < 1: + found.extend( + problem( + (*loc, position), + f"expected a positive [size, count] pair, got {item!r}", + "invalid_value", + ) + ) + return tuple(found) + + def configuration(self) -> dict[str, object]: + """Run-length encoded, which is the spelling that does not grow. + + Two dimension specs listing the same extents describe the same + grid, and the encoded one stays the same size as the array grows. + """ + members = super().configuration() + members["chunk_shapes"] = canonical_chunk_shapes(self.chunk_shapes) + return members + + def to_json(self) -> RectilinearChunkGridObject: + return cast("RectilinearChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index bdcd9e06c5..fb1a172bf5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -4,10 +4,19 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + MemberTypes, + MetadataEntity, + is_int, + sequence_of, +) + REGULAR_CHUNK_GRID_NAME: Final = "regular" """The `name` field value of the regular chunk grid.""" @@ -40,8 +49,42 @@ class RegularChunkGridObject(TypedDict, closed=True): __all__ = [ "REGULAR_CHUNK_GRID_NAME", + "RegularChunkGrid", "RegularChunkGridConfiguration", "RegularChunkGridMetadata", "RegularChunkGridName", "RegularChunkGridObject", ] + + +@dataclass(frozen=True) +class RegularChunkGrid(MetadataEntity): + """The `regular` chunk grid, coerced from its metadata.""" + + chunk_shape: tuple[int, ...] = () + + identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = {"chunk_shape": (True, sequence_of(is_int))} + + def problems(self) -> tuple[ValidationProblem, ...]: + """Every chunk extent must be at least one element. + + A chunk of zero elements along an axis covers nothing, so no + finite number of them tiles the axis; a negative one is + meaningless. Whether there is one extent *per array dimension* is + a question for the document, and the rules layer asks it. + """ + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(self.chunk_shape) + if extent < 1 + ) + + def to_json(self) -> RegularChunkGridObject: + return cast("RegularChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 10a7c0cb55..793482aa8e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -7,10 +7,17 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.v3._entity import ( + MemberTypes, + MetadataEntity, + one_of, +) + DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" """The `name` field value of the default chunk key encoding.""" @@ -55,9 +62,28 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): __all__ = [ "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", + "DefaultChunkKeyEncoding", "DefaultChunkKeyEncodingConfiguration", "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingObject", "DefaultChunkKeyEncodingSeparator", ] + + +@dataclass(frozen=True) +class DefaultChunkKeyEncoding(MetadataEntity): + """The `default` chunk key encoding, coerced from its metadata.""" + + separator: DefaultChunkKeyEncodingSeparator | None = None + + identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME + + member_types: ClassVar[MemberTypes] = { + "separator": (False, one_of(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)) + } + + def to_json(self) -> DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName: + return cast( + "DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName", super().to_json() + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 63cdf26783..7bcae4613e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -13,10 +13,17 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-key-encoding """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict +from zarr_metadata.v3._entity import ( + MemberTypes, + MetadataEntity, + one_of, +) + V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" """The `name` field value of the v2 chunk key encoding.""" @@ -61,9 +68,26 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): __all__ = [ "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", + "V2ChunkKeyEncoding", "V2ChunkKeyEncodingConfiguration", "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingObject", "V2ChunkKeyEncodingSeparator", ] + + +@dataclass(frozen=True) +class V2ChunkKeyEncoding(MetadataEntity): + """The `v2` chunk key encoding, coerced from its metadata.""" + + separator: V2ChunkKeyEncodingSeparator | None = None + + identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME + + member_types: ClassVar[MemberTypes] = { + "separator": (False, one_of(V2_CHUNK_KEY_ENCODING_SEPARATOR)) + } + + def to_json(self) -> V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName: + return cast("V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName", super().to_json()) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 95d0cec400..97aa04d570 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -15,6 +15,19 @@ import pytest from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS +from zarr_metadata.v3.chunk_grid.rectilinear import ( + RectilinearChunkGrid, + RectilinearChunkGridConfiguration, +) +from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid, RegularChunkGridConfiguration +from zarr_metadata.v3.chunk_key_encoding.default import ( + DefaultChunkKeyEncoding, + DefaultChunkKeyEncodingConfiguration, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2ChunkKeyEncoding, + V2ChunkKeyEncodingConfiguration, +) from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration from zarr_metadata.v3.codec.bytes import BytesCodec, BytesCodecConfiguration from zarr_metadata.v3.codec.crc32c import Crc32cCodec, Empty @@ -35,6 +48,10 @@ "scale_offset": (ScaleOffsetCodec, ScaleOffsetCodecConfiguration), "transpose": (TransposeCodec, TransposeCodecConfiguration), "zstd": (ZstdCodec, ZstdCodecConfiguration), + "regular": (RegularChunkGrid, RegularChunkGridConfiguration), + "rectilinear": (RectilinearChunkGrid, RectilinearChunkGridConfiguration), + "default": (DefaultChunkKeyEncoding, DefaultChunkKeyEncodingConfiguration), + "v2": (V2ChunkKeyEncoding, V2ChunkKeyEncodingConfiguration), } From ca740be2c6159f279afce6541380817b3b1c78c5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 16:59:43 +0200 Subject: [PATCH 027/107] feat(zarr-metadata): the data types, as entities Seventeen of the nineteen are a name and nothing else, so they are a name and nothing else here: three lines each, next to the constant that spells them. `r` is the one entity whose identifier is not a name any document carries, because the family is a shape rather than a spelling. It keeps the spelling it was given rather than the bit count it means, so `r008` comes back out as `r008` -- that is a valid way of writing eight bits, and rewriting it is not this package's call. `RAW_BYTES_FAMILY` moves to `data_type.raw`, which owns the grammar it names; `_extension_points` imports it alongside the pattern it already imported, and the cycle that would otherwise close does not. `Context` moves to `_registry`, where the mapping it wraps already lived. Keeping it next to `MetadataEntity` meant `_entity` importing `_extension_points`, which imports `data_type.raw`, which every data type module now imports back. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 25 +---- .../src/zarr_metadata/v3/_extension_points.py | 9 +- .../src/zarr_metadata/v3/_registry.py | 79 +++++++++++++- .../src/zarr_metadata/v3/data_type/bool.py | 13 ++- .../src/zarr_metadata/v3/data_type/bytes.py | 13 ++- .../zarr_metadata/v3/data_type/complex128.py | 12 ++- .../zarr_metadata/v3/data_type/complex64.py | 12 ++- .../src/zarr_metadata/v3/data_type/float16.py | 13 ++- .../src/zarr_metadata/v3/data_type/float32.py | 13 ++- .../src/zarr_metadata/v3/data_type/float64.py | 13 ++- .../src/zarr_metadata/v3/data_type/int16.py | 13 ++- .../src/zarr_metadata/v3/data_type/int32.py | 13 ++- .../src/zarr_metadata/v3/data_type/int64.py | 13 ++- .../src/zarr_metadata/v3/data_type/int8.py | 13 ++- .../v3/data_type/numpy_datetime64.py | 50 ++++++++- .../v3/data_type/numpy_timedelta64.py | 50 ++++++++- .../src/zarr_metadata/v3/data_type/raw.py | 79 +++++++++++++- .../src/zarr_metadata/v3/data_type/string.py | 13 ++- .../src/zarr_metadata/v3/data_type/uint16.py | 13 ++- .../src/zarr_metadata/v3/data_type/uint32.py | 13 ++- .../src/zarr_metadata/v3/data_type/uint64.py | 13 ++- .../src/zarr_metadata/v3/data_type/uint8.py | 13 ++- .../zarr-metadata/tests/v3/test_entities.py | 100 ++++++++++++++---- 23 files changed, 526 insertions(+), 72 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 3982a7535a..721f450aec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -45,7 +45,6 @@ from typing_extensions import TypeIs from zarr_metadata.model._validation import ValidationProblem, is_json -from zarr_metadata.v3._extension_points import canonical_name if TYPE_CHECKING: from collections.abc import Callable, Mapping @@ -53,7 +52,7 @@ from zarr_metadata.model._validation import ProblemKind from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - from zarr_metadata.v3._extension_points import ExtensionPointField + from zarr_metadata.v3._registry import Context EntityT = TypeVar("EntityT", bound="MetadataEntity") @@ -200,27 +199,6 @@ def coerce_members( return members, tuple(problems) -@dataclass(frozen=True, slots=True) -class Context: - """The entities in scope while metadata is being read. - - A scope is not a property of the entities, it is a choice the reader - makes: judging against the specification alone, or against the - specification plus what `zarr-extensions` registers. The two live in - `zarr_metadata.v3._registry`. - """ - - entities: Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]] - - def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: - """The entity `name` denotes at `field`, or None if out of scope. - - Out of scope is not an error: an unknown name may be an extension - this reader does not model, and openness means leaving it unjudged. - """ - return self.entities.get(field, {}).get(canonical_name(field, name)) - - # No `slots=True`, deliberately: it rebuilds the class, which leaves the # zero-argument `super()` in a subclass pointing at the class that was # replaced. Subclasses call `super()` to narrow `to_json` and to adjust @@ -374,7 +352,6 @@ def named_configuration( __all__ = [ "CodecKind", "Coerced", - "Context", "Loc", "MemberTypes", "MetadataEntity", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py index 631a9e705f..22bb0b2cec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py @@ -14,7 +14,7 @@ from typing import Final, Literal -from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN +from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY, RAW_BYTES_NAME_PATTERN ExtensionPointField = Literal[ "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" @@ -26,13 +26,6 @@ CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" CODECS: Final[ExtensionPointField] = "codecs" -RAW_BYTES_FAMILY: Final = "r" -"""Canonical key for the parameterized raw-bytes data type family. - -Spelled as the spec writes the family; the angle brackets keep it -unforgeable by a real name. -""" - def canonical_name(field: ExtensionPointField, name: str) -> str: """`name` reduced to the key this package tables it under.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 3d763f21cc..2ab4a5a0aa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -20,14 +20,15 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING, Final -from zarr_metadata.v3._entity import Context from zarr_metadata.v3._extension_points import ( CHUNK_GRID, CHUNK_KEY_ENCODING, CODECS, DATA_TYPE, + canonical_name, ) from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid @@ -40,9 +41,59 @@ from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec from zarr_metadata.v3.codec.transpose import TransposeCodec from zarr_metadata.v3.codec.zstd import ZstdCodec +from zarr_metadata.v3.data_type.bool import BoolDataType +from zarr_metadata.v3.data_type.bytes import BytesDataType +from zarr_metadata.v3.data_type.complex64 import Complex64DataType +from zarr_metadata.v3.data_type.complex128 import Complex128DataType +from zarr_metadata.v3.data_type.float16 import Float16DataType +from zarr_metadata.v3.data_type.float32 import Float32DataType +from zarr_metadata.v3.data_type.float64 import Float64DataType +from zarr_metadata.v3.data_type.int8 import Int8DataType +from zarr_metadata.v3.data_type.int16 import Int16DataType +from zarr_metadata.v3.data_type.int32 import Int32DataType +from zarr_metadata.v3.data_type.int64 import Int64DataType +from zarr_metadata.v3.data_type.numpy_datetime64 import NumpyDatetime64DataType +from zarr_metadata.v3.data_type.numpy_timedelta64 import NumpyTimedelta64DataType +from zarr_metadata.v3.data_type.raw import RawBytesDataType +from zarr_metadata.v3.data_type.string import StringDataType +from zarr_metadata.v3.data_type.uint8 import Uint8DataType +from zarr_metadata.v3.data_type.uint16 import Uint16DataType +from zarr_metadata.v3.data_type.uint32 import Uint32DataType +from zarr_metadata.v3.data_type.uint64 import Uint64DataType if TYPE_CHECKING: + from collections.abc import Mapping + from zarr_metadata.v3._entity import MetadataEntity + from zarr_metadata.v3._extension_points import ExtensionPointField + + +@dataclass(frozen=True, slots=True) +class Context: + """The entities in scope while metadata is being read. + + Passed to every `coerce`, and most entities ignore it: a `gzip` codec + is a `gzip` codec whatever else is in scope. The ones that do not + ignore it hold other entities inside their own configuration -- a + `struct` data type holds field data types, a `sharding_indexed` codec + holds two codec pipelines -- and cannot read those without knowing + what is in scope inside them. + + A scope is not a property of the entities, it is a choice the reader + makes: judging against the specification alone, or against the + specification plus what `zarr-extensions` registers. + """ + + entities: Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]] + + def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: + """The entity `name` denotes at `field`, or None if out of scope. + + Out of scope is not an error: an unknown name may be an extension + this reader does not model, and openness means leaving it unjudged. + """ + return self.entities.get(field, {}).get(canonical_name(field, name)) + _CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { BloscCodec.identifier: BloscCodec, @@ -56,8 +107,29 @@ ZstdCodec.identifier: ZstdCodec, } -_CORE_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} -_EXTENSION_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = {} +_CORE_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = { + BoolDataType.identifier: BoolDataType, + Int8DataType.identifier: Int8DataType, + Int16DataType.identifier: Int16DataType, + Int32DataType.identifier: Int32DataType, + Int64DataType.identifier: Int64DataType, + Uint8DataType.identifier: Uint8DataType, + Uint16DataType.identifier: Uint16DataType, + Uint32DataType.identifier: Uint32DataType, + Uint64DataType.identifier: Uint64DataType, + Float16DataType.identifier: Float16DataType, + Float32DataType.identifier: Float32DataType, + Float64DataType.identifier: Float64DataType, + Complex64DataType.identifier: Complex64DataType, + Complex128DataType.identifier: Complex128DataType, + RawBytesDataType.identifier: RawBytesDataType, +} +_EXTENSION_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = { + BytesDataType.identifier: BytesDataType, + StringDataType.identifier: StringDataType, + NumpyDatetime64DataType.identifier: NumpyDatetime64DataType, + NumpyTimedelta64DataType.identifier: NumpyTimedelta64DataType, +} _CORE_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = { RegularChunkGrid.identifier: RegularChunkGrid, @@ -96,4 +168,5 @@ __all__ = [ "CORE", "CORE_AND_EXTENSIONS", + "Context", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index e36613a154..f5d1168b82 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity BOOL_DATA_TYPE_NAME: Final = "bool" """The `data_type` value for the `bool` type.""" @@ -18,6 +21,14 @@ __all__ = [ "BOOL_DATA_TYPE_NAME", + "BoolDataType", "BoolDataTypeName", "BoolFillValue", ] + + +@dataclass(frozen=True) +class BoolDataType(MetadataEntity): + """The `bool` data type. The name says everything.""" + + identifier: ClassVar[str] = BOOL_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 5892a1bdaa..09040de036 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -5,7 +5,10 @@ """ import re -from typing import Final, Literal, NewType +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NewType + +from zarr_metadata.v3._entity import MetadataEntity BYTES_DATA_TYPE_NAME: Final = "bytes" """The `data_type` value for the variable-length `bytes` type.""" @@ -42,7 +45,15 @@ def base64_bytes(value: str) -> Base64Bytes: __all__ = [ "BYTES_DATA_TYPE_NAME", "Base64Bytes", + "BytesDataType", "BytesDataTypeName", "BytesFillValue", "base64_bytes", ] + + +@dataclass(frozen=True) +class BytesDataType(MetadataEntity): + """The `bytes` data type. The name says everything.""" + + identifier: ClassVar[str] = BYTES_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 780bbbb02f..e800efd960 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -4,8 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal +from zarr_metadata.v3._entity import MetadataEntity from zarr_metadata.v3.data_type.float64 import Float64FillValue COMPLEX128_DATA_TYPE_NAME: Final = "complex128" @@ -32,6 +34,14 @@ __all__ = [ "COMPLEX128_DATA_TYPE_NAME", "Complex128Component", + "Complex128DataType", "Complex128DataTypeName", "Complex128FillValue", ] + + +@dataclass(frozen=True) +class Complex128DataType(MetadataEntity): + """The `complex128` data type. The name says everything.""" + + identifier: ClassVar[str] = COMPLEX128_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 4aca608899..3d1b8d42c7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -4,8 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal +from zarr_metadata.v3._entity import MetadataEntity from zarr_metadata.v3.data_type.float32 import Float32FillValue COMPLEX64_DATA_TYPE_NAME: Final = "complex64" @@ -32,6 +34,14 @@ __all__ = [ "COMPLEX64_DATA_TYPE_NAME", "Complex64Component", + "Complex64DataType", "Complex64DataTypeName", "Complex64FillValue", ] + + +@dataclass(frozen=True) +class Complex64DataType(MetadataEntity): + """The `complex64` data type. The name says everything.""" + + identifier: ClassVar[str] = COMPLEX64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 264b2c262c..970b0aa536 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -5,7 +5,10 @@ """ import re -from typing import Final, Literal, NewType +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NewType + +from zarr_metadata.v3._entity import MetadataEntity FLOAT16_DATA_TYPE_NAME: Final = "float16" """The `data_type` value for the `float16` type.""" @@ -66,9 +69,17 @@ def hex_float16(value: str) -> HexFloat16: "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT16", "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT16", "FLOAT16_DATA_TYPE_NAME", + "Float16DataType", "Float16DataTypeName", "Float16FillValue", "Float16SpecialFillValue", "HexFloat16", "hex_float16", ] + + +@dataclass(frozen=True) +class Float16DataType(MetadataEntity): + """The `float16` data type. The name says everything.""" + + identifier: ClassVar[str] = FLOAT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 3b2e786f07..594dcfce0a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -5,7 +5,10 @@ """ import re -from typing import Final, Literal, NewType +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NewType + +from zarr_metadata.v3._entity import MetadataEntity FLOAT32_DATA_TYPE_NAME: Final = "float32" """The `data_type` value for the `float32` type.""" @@ -66,9 +69,17 @@ def hex_float32(value: str) -> HexFloat32: "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT32", "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT32", "FLOAT32_DATA_TYPE_NAME", + "Float32DataType", "Float32DataTypeName", "Float32FillValue", "Float32SpecialFillValue", "HexFloat32", "hex_float32", ] + + +@dataclass(frozen=True) +class Float32DataType(MetadataEntity): + """The `float32` data type. The name says everything.""" + + identifier: ClassVar[str] = FLOAT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 21373d63f6..dda2be3a63 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -5,7 +5,10 @@ """ import re -from typing import Final, Literal, NewType +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NewType + +from zarr_metadata.v3._entity import MetadataEntity FLOAT64_DATA_TYPE_NAME: Final = "float64" """The `data_type` value for the `float64` type.""" @@ -67,9 +70,17 @@ def hex_float64(value: str) -> HexFloat64: "CANONICAL_NEGATIVE_INFINITY_HEX_FLOAT64", "CANONICAL_POSITIVE_INFINITY_HEX_FLOAT64", "FLOAT64_DATA_TYPE_NAME", + "Float64DataType", "Float64DataTypeName", "Float64FillValue", "Float64SpecialFillValue", "HexFloat64", "hex_float64", ] + + +@dataclass(frozen=True) +class Float64DataType(MetadataEntity): + """The `float64` data type. The name says everything.""" + + identifier: ClassVar[str] = FLOAT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index b76f06761a..145573f27d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity INT16_DATA_TYPE_NAME: Final = "int16" """The `data_type` value for the `int16` type.""" @@ -18,6 +21,14 @@ __all__ = [ "INT16_DATA_TYPE_NAME", + "Int16DataType", "Int16DataTypeName", "Int16FillValue", ] + + +@dataclass(frozen=True) +class Int16DataType(MetadataEntity): + """The `int16` data type. The name says everything.""" + + identifier: ClassVar[str] = INT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 7b41ec6c54..3a27176d4f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity INT32_DATA_TYPE_NAME: Final = "int32" """The `data_type` value for the `int32` type.""" @@ -18,6 +21,14 @@ __all__ = [ "INT32_DATA_TYPE_NAME", + "Int32DataType", "Int32DataTypeName", "Int32FillValue", ] + + +@dataclass(frozen=True) +class Int32DataType(MetadataEntity): + """The `int32` data type. The name says everything.""" + + identifier: ClassVar[str] = INT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index 0005675c66..293da17f7b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity INT64_DATA_TYPE_NAME: Final = "int64" """The `data_type` value for the `int64` type.""" @@ -18,6 +21,14 @@ __all__ = [ "INT64_DATA_TYPE_NAME", + "Int64DataType", "Int64DataTypeName", "Int64FillValue", ] + + +@dataclass(frozen=True) +class Int64DataType(MetadataEntity): + """The `int64` data type. The name says everything.""" + + identifier: ClassVar[str] = INT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index a5a16de761..875e775750 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity INT8_DATA_TYPE_NAME: Final = "int8" """The `data_type` value for the `int8` type.""" @@ -18,6 +21,14 @@ __all__ = [ "INT8_DATA_TYPE_NAME", + "Int8DataType", "Int8DataTypeName", "Int8FillValue", ] + + +@dataclass(frozen=True) +class Int8DataType(MetadataEntity): + """The `int8` data type. The name says everything.""" + + identifier: ClassVar[str] = INT8_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 4f9a6415c5..1c8364328f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -4,10 +4,24 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import ReadOnly, TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + MemberTypes, + MetadataEntity, + is_int, + one_of, + problem, +) +from zarr_metadata.v3.data_type.numpy_timedelta64 import ( + NUMPY_TIME_MAX_SCALE_FACTOR, + NUMPY_TIME_UNIT, +) + NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" """The `name` field value of the `numpy.datetime64` data type.""" @@ -55,7 +69,41 @@ class NumpyDatetime64(TypedDict, closed=True): "NUMPY_DATETIME64_DATA_TYPE_NAME", "NumpyDatetime64", "NumpyDatetime64Configuration", + "NumpyDatetime64DataType", "NumpyDatetime64DataTypeName", "NumpyDatetime64FillValue", "NumpyTimeUnit", ] + + +@dataclass(frozen=True) +class NumpyDatetime64DataType(MetadataEntity): + """The `numpy.datetime64` data type, coerced from its metadata.""" + + unit: NumpyTimeUnit = "generic" + scale_factor: int = 1 + + identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { + "unit": (True, one_of(NUMPY_TIME_UNIT)), + "scale_factor": (True, is_int), + } + + def problems(self) -> tuple[ValidationProblem, ...]: + """`scale_factor` counts units per step, so it is positive. + + The upper bound is numpy's: the field is a signed 32-bit integer. + """ + if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + return problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " + f"got {self.scale_factor}", + "invalid_value", + ) + return () + + def to_json(self) -> NumpyDatetime64: + return cast("NumpyDatetime64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index a1c4fef772..55a57aea45 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -4,10 +4,20 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ -from typing import Final, Literal, NotRequired +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast from typing_extensions import ReadOnly, TypedDict +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + MemberTypes, + MetadataEntity, + is_int, + one_of, + problem, +) + NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" """The `name` field value of the `numpy.timedelta64` data type.""" @@ -19,6 +29,9 @@ ] """Time unit codes used by numpy.timedelta64.""" +NUMPY_TIME_MAX_SCALE_FACTOR: Final = 2**31 - 1 +"""The largest `scale_factor` numpy stores: the field is a signed int32.""" + NUMPY_TIME_UNIT: Final = ( "Y", "M", @@ -72,10 +85,45 @@ class NumpyTimedelta64(TypedDict, closed=True): __all__ = [ "NUMPY_TIMEDELTA64_DATA_TYPE_NAME", + "NUMPY_TIME_MAX_SCALE_FACTOR", "NUMPY_TIME_UNIT", "NumpyTimeUnit", "NumpyTimedelta64", "NumpyTimedelta64Configuration", + "NumpyTimedelta64DataType", "NumpyTimedelta64DataTypeName", "NumpyTimedelta64FillValue", ] + + +@dataclass(frozen=True) +class NumpyTimedelta64DataType(MetadataEntity): + """The `numpy.timedelta64` data type, coerced from its metadata.""" + + unit: NumpyTimeUnit = "generic" + scale_factor: int = 1 + + identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { + "unit": (True, one_of(NUMPY_TIME_UNIT)), + "scale_factor": (True, is_int), + } + + def problems(self) -> tuple[ValidationProblem, ...]: + """`scale_factor` counts units per step, so it is positive. + + The upper bound is numpy's: the field is a signed 32-bit integer. + """ + if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + return problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " + f"got {self.scale_factor}", + "invalid_value", + ) + return () + + def to_json(self) -> NumpyTimedelta64: + return cast("NumpyTimedelta64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 369dc282c1..5f35bbba21 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -9,7 +9,17 @@ """ import re -from typing import Final, NewType +from dataclasses import dataclass +from typing import ClassVar, Final, NewType, Self, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3._entity import ( + Coerced, + MetadataEntity, + named_configuration, + problem, +) RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) """A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). @@ -18,6 +28,13 @@ https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 """ +RAW_BYTES_FAMILY: Final = "r" +"""Canonical key for the parameterized raw-bytes data type family. + +Spelled as the spec writes the family; the angle brackets keep it +unforgeable by a real name. +""" + RAW_BYTES_NAME_PATTERN: Final = re.compile(r"^r(\d+)$") """The *shape* of a raw-bytes data type name, not its validity. @@ -51,8 +68,68 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: __all__ = [ + "RAW_BYTES_FAMILY", "RAW_BYTES_NAME_PATTERN", + "RawBytesDataType", "RawBytesDataTypeName", "RawBytesFillValue", "raw_bytes_dtype_name", ] + + +@dataclass(frozen=True) +class RawBytesDataType(MetadataEntity): + """An `r` raw-bytes data type, coerced from its metadata. + + One class for the whole family, because `r8` and `r4096` differ only + in a number. That is why this is the one entity whose `identifier` is + not a name any document carries: `r` is a shape, not a spelling, + and no real name can collide with it. + + The spelling is kept rather than the bit count, so a document comes + back out as it went in. `r008` is a valid and distinct way of writing + `r8`, and canonicalizing it away is not this package's call. + """ + + data_type_name: str = "r8" + + identifier: ClassVar[str] = RAW_BYTES_FAMILY + + @classmethod + def accepts(cls, name: str) -> bool: + """Every `r` spelling, valid or not. + + A malformed member of the family is recognized as belonging to it + and reported as malformed, rather than passing unjudged as some + third party's extension. + """ + return RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None + + @classmethod + def coerce(cls, value: object, context: object) -> Coerced[Self]: + name, configuration, must_understand = named_configuration(value) + if name is None or not cls.accepts(name): + return None, problem((), "expected an 'r' raw-bytes data type") + if configuration is not None and len(configuration) != 0: + return None, problem(("configuration",), "'r' takes no configuration", "unknown_key") + return cls(must_understand=must_understand, data_type_name=name), () + + def problems(self) -> tuple[ValidationProblem, ...]: + """N must be a positive multiple of 8. + + "raw bits, variable size given by *, limited to be a multiple of + 8" -- and zero bits is not a data type. + """ + try: + raw_bytes_dtype_name(self.data_type_name) + except ValueError as error: + return problem((), str(error), "invalid_value") + return () + + def to_json(self) -> ZarrV3MetadataFieldJSON: + if self.must_understand: + return cast("ZarrV3MetadataFieldJSON", self.data_type_name) + return cast( + "ZarrV3MetadataFieldJSON", + {"name": self.data_type_name, "must_understand": False}, + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 1e93a95d50..04fdccd3e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -4,7 +4,10 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity STRING_DATA_TYPE_NAME: Final = "string" """The `data_type` value for the `string` type.""" @@ -18,6 +21,14 @@ __all__ = [ "STRING_DATA_TYPE_NAME", + "StringDataType", "StringDataTypeName", "StringFillValue", ] + + +@dataclass(frozen=True) +class StringDataType(MetadataEntity): + """The `string` data type. The name says everything.""" + + identifier: ClassVar[str] = STRING_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 37e35ec436..aa2b5dc07c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity UINT16_DATA_TYPE_NAME: Final = "uint16" """The `data_type` value for the `uint16` type.""" @@ -18,6 +21,14 @@ __all__ = [ "UINT16_DATA_TYPE_NAME", + "Uint16DataType", "Uint16DataTypeName", "Uint16FillValue", ] + + +@dataclass(frozen=True) +class Uint16DataType(MetadataEntity): + """The `uint16` data type. The name says everything.""" + + identifier: ClassVar[str] = UINT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index f6cd4d447e..fded4f2ab6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity UINT32_DATA_TYPE_NAME: Final = "uint32" """The `data_type` value for the `uint32` type.""" @@ -18,6 +21,14 @@ __all__ = [ "UINT32_DATA_TYPE_NAME", + "Uint32DataType", "Uint32DataTypeName", "Uint32FillValue", ] + + +@dataclass(frozen=True) +class Uint32DataType(MetadataEntity): + """The `uint32` data type. The name says everything.""" + + identifier: ClassVar[str] = UINT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 7151d2395a..e06a8db7c4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity UINT64_DATA_TYPE_NAME: Final = "uint64" """The `data_type` value for the `uint64` type.""" @@ -18,6 +21,14 @@ __all__ = [ "UINT64_DATA_TYPE_NAME", + "Uint64DataType", "Uint64DataTypeName", "Uint64FillValue", ] + + +@dataclass(frozen=True) +class Uint64DataType(MetadataEntity): + """The `uint64` data type. The name says everything.""" + + identifier: ClassVar[str] = UINT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index 787f1b7866..7a1f6a3e85 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -4,7 +4,10 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from typing import Final, Literal +from dataclasses import dataclass +from typing import ClassVar, Final, Literal + +from zarr_metadata.v3._entity import MetadataEntity UINT8_DATA_TYPE_NAME: Final = "uint8" """The `data_type` value for the `uint8` type.""" @@ -18,6 +21,14 @@ __all__ = [ "UINT8_DATA_TYPE_NAME", + "Uint8DataType", "Uint8DataTypeName", "Uint8FillValue", ] + + +@dataclass(frozen=True) +class Uint8DataType(MetadataEntity): + """The `uint8` data type. The name says everything.""" + + identifier: ClassVar[str] = UINT8_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 97aa04d570..d2e0025bc4 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -35,23 +35,75 @@ from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec, ScaleOffsetCodecConfiguration from zarr_metadata.v3.codec.transpose import TransposeCodec, TransposeCodecConfiguration from zarr_metadata.v3.codec.zstd import ZstdCodec, ZstdCodecConfiguration +from zarr_metadata.v3.data_type.bool import BoolDataType +from zarr_metadata.v3.data_type.bytes import BytesDataType +from zarr_metadata.v3.data_type.complex64 import Complex64DataType +from zarr_metadata.v3.data_type.complex128 import Complex128DataType +from zarr_metadata.v3.data_type.float16 import Float16DataType +from zarr_metadata.v3.data_type.float32 import Float32DataType +from zarr_metadata.v3.data_type.float64 import Float64DataType +from zarr_metadata.v3.data_type.int8 import Int8DataType +from zarr_metadata.v3.data_type.int16 import Int16DataType +from zarr_metadata.v3.data_type.int32 import Int32DataType +from zarr_metadata.v3.data_type.int64 import Int64DataType +from zarr_metadata.v3.data_type.numpy_datetime64 import ( + NumpyDatetime64Configuration, + NumpyDatetime64DataType, +) +from zarr_metadata.v3.data_type.numpy_timedelta64 import ( + NumpyTimedelta64Configuration, + NumpyTimedelta64DataType, +) +from zarr_metadata.v3.data_type.raw import RawBytesDataType +from zarr_metadata.v3.data_type.string import StringDataType +from zarr_metadata.v3.data_type.uint8 import Uint8DataType +from zarr_metadata.v3.data_type.uint16 import Uint16DataType +from zarr_metadata.v3.data_type.uint32 import Uint32DataType +from zarr_metadata.v3.data_type.uint64 import Uint64DataType if TYPE_CHECKING: from zarr_metadata.v3._entity import MetadataEntity -# Each registered entity, paired with the TypedDict its constructor mirrors. -CONFIGURATIONS: dict[str, tuple[type[MetadataEntity], type]] = { - "blosc": (BloscCodec, BloscCodecConfiguration), - "bytes": (BytesCodec, BytesCodecConfiguration), - "crc32c": (Crc32cCodec, Empty), - "gzip": (GzipCodec, GzipCodecConfiguration), - "scale_offset": (ScaleOffsetCodec, ScaleOffsetCodecConfiguration), - "transpose": (TransposeCodec, TransposeCodecConfiguration), - "zstd": (ZstdCodec, ZstdCodecConfiguration), - "regular": (RegularChunkGrid, RegularChunkGridConfiguration), - "rectilinear": (RectilinearChunkGrid, RectilinearChunkGridConfiguration), - "default": (DefaultChunkKeyEncoding, DefaultChunkKeyEncodingConfiguration), - "v2": (V2ChunkKeyEncoding, V2ChunkKeyEncodingConfiguration), +# Each registered entity, paired with the TypedDict its constructor +# mirrors. Keyed by `:`, because an identifier is only +# unique within its extension point -- `bytes` is both a codec and a data +# type. +CONFIGURATIONS: dict[str, tuple[type[MetadataEntity], type | None]] = { + "codecs:blosc": (BloscCodec, BloscCodecConfiguration), + "codecs:bytes": (BytesCodec, BytesCodecConfiguration), + "codecs:crc32c": (Crc32cCodec, Empty), + "codecs:gzip": (GzipCodec, GzipCodecConfiguration), + "codecs:scale_offset": (ScaleOffsetCodec, ScaleOffsetCodecConfiguration), + "codecs:transpose": (TransposeCodec, TransposeCodecConfiguration), + "codecs:zstd": (ZstdCodec, ZstdCodecConfiguration), + "chunk_grid:regular": (RegularChunkGrid, RegularChunkGridConfiguration), + "chunk_grid:rectilinear": (RectilinearChunkGrid, RectilinearChunkGridConfiguration), + "chunk_key_encoding:default": (DefaultChunkKeyEncoding, DefaultChunkKeyEncodingConfiguration), + "chunk_key_encoding:v2": (V2ChunkKeyEncoding, V2ChunkKeyEncodingConfiguration), + "data_type:numpy.datetime64": (NumpyDatetime64DataType, NumpyDatetime64Configuration), + "data_type:numpy.timedelta64": (NumpyTimedelta64DataType, NumpyTimedelta64Configuration), + # Bare entities: the name says everything, so there is no + # configuration TypedDict for the constructor to mirror. + "data_type:bool": (BoolDataType, None), + "data_type:int8": (Int8DataType, None), + "data_type:int16": (Int16DataType, None), + "data_type:int32": (Int32DataType, None), + "data_type:int64": (Int64DataType, None), + "data_type:uint8": (Uint8DataType, None), + "data_type:uint16": (Uint16DataType, None), + "data_type:uint32": (Uint32DataType, None), + "data_type:uint64": (Uint64DataType, None), + "data_type:float16": (Float16DataType, None), + "data_type:float32": (Float32DataType, None), + "data_type:float64": (Float64DataType, None), + "data_type:complex64": (Complex64DataType, None), + "data_type:complex128": (Complex128DataType, None), + "data_type:bytes": (BytesDataType, None), + "data_type:string": (StringDataType, None), + # The one exception. `r` is a family, so the class holds the + # spelling that picks a member of it -- a field with no configuration + # member behind it, because the name carries the information. + "data_type:r": (RawBytesDataType, None), } @@ -59,10 +111,16 @@ ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) ) def test_the_constructor_mirrors_the_configuration( - entity: type[MetadataEntity], configuration: type + entity: type[MetadataEntity], configuration: type | None ) -> None: # `must_understand` belongs to the object, not the configuration, so it # is the one field the two deliberately do not share. + if configuration is None: + expected = {"data_type_name"} if entity is RawBytesDataType else set() + assert {field.name for field in dataclasses.fields(entity)} - { + "must_understand" + } == expected + return fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} assert fields == set(get_type_hints(configuration)) @@ -71,10 +129,13 @@ def test_the_constructor_mirrors_the_configuration( ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) ) def test_the_member_table_mirrors_the_configuration( - entity: type[MetadataEntity], configuration: type + entity: type[MetadataEntity], configuration: type | None ) -> None: # The third spelling of the same set. Which members are *required* is # in the TypedDict too, so that cannot drift either. + if configuration is None: + assert entity.member_types == {} + return assert set(entity.member_types) == set(get_type_hints(configuration)) required = {key for key, (needed, _) in entity.member_types.items() if needed} assert required == set(configuration.__required_keys__) # type: ignore[attr-defined] @@ -84,16 +145,19 @@ def test_the_member_table_mirrors_the_configuration( ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) ) def test_a_required_member_rules_out_the_bare_spelling( - entity: type[MetadataEntity], configuration: type + entity: type[MetadataEntity], configuration: type | None ) -> None: # The spec permits a bare name only "if no configuration metadata is # required", so one flag follows from the other. - assert entity.configuration_required == (len(configuration.__required_keys__) != 0) # type: ignore[attr-defined] + required = 0 if configuration is None else len(configuration.__required_keys__) # type: ignore[attr-defined] + assert entity.configuration_required == (required != 0) def test_every_registered_entity_is_checked_here() -> None: registered = { - identifier for entities in CORE_AND_EXTENSIONS.entities.values() for identifier in entities + f"{field}:{identifier}" + for field, entities in CORE_AND_EXTENSIONS.entities.items() + for identifier in entities } assert registered == set(CONFIGURATIONS) From d3d435b05943b69fe74b0914685ab81136b611f4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:05:44 +0200 Subject: [PATCH 028/107] feat(zarr-metadata): the entities that contain other entities `struct`, `sharding_indexed` and `cast_value` hold entities inside their own configuration, and `Context.coerce` is the primitive they are built from: it reads one nested entity in the scope the outer one was read in, returning the value untouched when its name is out of scope, and prefixing the problems with where in the configuration it sat. A nested codec's bad compression level now reports at `codecs.1.level`. Storage class becomes polymorphism. `_storage_class`'s three name sets said which data types are single-byte, multi-byte or variable-length, and hand-rolled the recursion for `struct`; now each data type declares its own and `StructDataType` folds its fields, which is the same recursion written once as a method call. The extension-point constants move to `_entity`. An entity that contains others has to name the point it reads them at, and `_extension_points` also folds `r` names -- which means importing the data types, which import the entity base. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 59 +++++- .../src/zarr_metadata/v3/_extension_points.py | 19 +- .../src/zarr_metadata/v3/_registry.py | 38 +++- .../src/zarr_metadata/v3/codec/cast_value.py | 116 ++++++++++- .../v3/codec/sharding_indexed.py | 124 +++++++++++- .../src/zarr_metadata/v3/data_type/bool.py | 5 +- .../src/zarr_metadata/v3/data_type/bytes.py | 5 +- .../zarr_metadata/v3/data_type/complex128.py | 5 +- .../zarr_metadata/v3/data_type/complex64.py | 5 +- .../src/zarr_metadata/v3/data_type/float16.py | 5 +- .../src/zarr_metadata/v3/data_type/float32.py | 5 +- .../src/zarr_metadata/v3/data_type/float64.py | 5 +- .../src/zarr_metadata/v3/data_type/int16.py | 5 +- .../src/zarr_metadata/v3/data_type/int32.py | 5 +- .../src/zarr_metadata/v3/data_type/int64.py | 5 +- .../src/zarr_metadata/v3/data_type/int8.py | 5 +- .../v3/data_type/numpy_datetime64.py | 6 +- .../v3/data_type/numpy_timedelta64.py | 6 +- .../src/zarr_metadata/v3/data_type/raw.py | 6 +- .../src/zarr_metadata/v3/data_type/string.py | 5 +- .../src/zarr_metadata/v3/data_type/struct.py | 184 +++++++++++++++++- .../src/zarr_metadata/v3/data_type/uint16.py | 5 +- .../src/zarr_metadata/v3/data_type/uint32.py | 5 +- .../src/zarr_metadata/v3/data_type/uint64.py | 5 +- .../src/zarr_metadata/v3/data_type/uint8.py | 5 +- .../zarr-metadata/tests/v3/test_entities.py | 24 +++ 26 files changed, 606 insertions(+), 56 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 721f450aec..3b684e1003 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -40,7 +40,7 @@ from collections.abc import Mapping as _Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Literal, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast from typing_extensions import TypeIs @@ -68,6 +68,30 @@ Loc: TypeAlias = "tuple[str | int, ...]" +ExtensionPointField = Literal[ + "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" +] +"""The v3 array metadata fields whose values name an extension. + +Here rather than in `_extension_points` because an entity that contains +other entities has to say which point it is reading them at, and +`_extension_points` also folds `r` names -- which means importing the +data types, which import this. +""" + +DATA_TYPE: Final[ExtensionPointField] = "data_type" +CHUNK_GRID: Final[ExtensionPointField] = "chunk_grid" +CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" +CODECS: Final[ExtensionPointField] = "codecs" + +StorageClass = Literal["single_byte", "multi_byte", "variable_length"] +"""How one scalar of a data type occupies bytes. + +`single_byte` and `multi_byte` are both fixed-size; they differ only in +whether a byte order applies, which is what the `bytes` codec's `endian` +member is about. +""" + CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] """The three pipeline positions the v3 spec sorts codecs into. @@ -194,7 +218,10 @@ def coerce_members( value = _as_tuples(configuration[key]) found = check(value, ("configuration", key)) problems.extend(found) - if len(found) == 0: + # An unknown key says the value carries something extra, not that + # it is the wrong type -- so the member is still readable, and + # dropping it here would make `to_json` lose what was written. + if all(entry.kind == "unknown_key" for entry in found): members[key] = value return members, tuple(problems) @@ -321,6 +348,27 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: return cast("ZarrV3MetadataFieldJSON", entry) +@dataclass(frozen=True) +class DataTypeEntity(MetadataEntity): + """An entity that says how the array's scalars are stored. + + Only data types answer that, and every rule that turns on it -- a + `bytes` codec is pointless before a single-byte type, a struct field + cannot be variable-length -- asks a data type rather than consulting + a table of names. + """ + + scalar_storage: ClassVar[StorageClass] + + def storage_class(self) -> StorageClass | None: + """How one scalar occupies bytes, or None if undetermined. + + None only for a composite whose parts are not all in scope: an + answer would be a guess, and the rules that ask decline instead. + """ + return type(self).scalar_storage + + def named_configuration( value: object, ) -> tuple[str | None, Mapping[str, object] | None, bool]: @@ -350,11 +398,18 @@ def named_configuration( __all__ = [ + "CHUNK_GRID", + "CHUNK_KEY_ENCODING", + "CODECS", + "DATA_TYPE", "CodecKind", "Coerced", + "DataTypeEntity", + "ExtensionPointField", "Loc", "MemberTypes", "MetadataEntity", + "StorageClass", "TypeCheck", "coerce_members", "is_bool", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py index 22bb0b2cec..0ed9ac77e0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py @@ -12,20 +12,15 @@ from __future__ import annotations -from typing import Final, Literal - +from zarr_metadata.v3._entity import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, + ExtensionPointField, +) from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY, RAW_BYTES_NAME_PATTERN -ExtensionPointField = Literal[ - "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" -] -"""The v3 array metadata fields whose values name an extension.""" - -DATA_TYPE: Final[ExtensionPointField] = "data_type" -CHUNK_GRID: Final[ExtensionPointField] = "chunk_grid" -CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" -CODECS: Final[ExtensionPointField] = "codecs" - def canonical_name(field: ExtensionPointField, name: str) -> str: """`name` reduced to the key this package tables it under.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 2ab4a5a0aa..e9c9ae79ec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -23,6 +23,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Final +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import named_configuration from zarr_metadata.v3._extension_points import ( CHUNK_GRID, CHUNK_KEY_ENCODING, @@ -36,9 +38,11 @@ from zarr_metadata.v3.chunk_key_encoding.v2 import V2ChunkKeyEncoding from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.bytes import BytesCodec +from zarr_metadata.v3.codec.cast_value import CastValueCodec from zarr_metadata.v3.codec.crc32c import Crc32cCodec from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec +from zarr_metadata.v3.codec.sharding_indexed import ShardingIndexedCodec from zarr_metadata.v3.codec.transpose import TransposeCodec from zarr_metadata.v3.codec.zstd import ZstdCodec from zarr_metadata.v3.data_type.bool import BoolDataType @@ -56,6 +60,7 @@ from zarr_metadata.v3.data_type.numpy_timedelta64 import NumpyTimedelta64DataType from zarr_metadata.v3.data_type.raw import RawBytesDataType from zarr_metadata.v3.data_type.string import StringDataType +from zarr_metadata.v3.data_type.struct import StructDataType from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType @@ -64,7 +69,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.v3._entity import MetadataEntity + from zarr_metadata.v3._entity import Loc, MetadataEntity from zarr_metadata.v3._extension_points import ExtensionPointField @@ -94,15 +99,45 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] """ return self.entities.get(field, {}).get(canonical_name(field, name)) + def coerce( + self, field: ExtensionPointField, value: object, loc: Loc = () + ) -> tuple[MetadataEntity | object, tuple[ValidationProblem, ...]]: + """One nested entity, read in this scope. + + The primitive the containing entities are built from: a `struct` + data type reads its fields with it, a `sharding_indexed` codec its + two pipelines. Returns the entity when its name is in scope, and + the value untouched when it is not -- an unmodelled extension is + left unjudged, which is what makes the format open. + + `loc` prefixes the problems, so they point at where in the + containing configuration the entity sat. + """ + name, _, _ = named_configuration(value) + if name is None: + return value, ( + ValidationProblem(loc, f"expected a metadata field, got {value!r}", "invalid_type"), + ) + entity_type = self.resolve(field, name) + if entity_type is None: + return value, () + entity, problems = entity_type.coerce(value, self) + located = tuple( + ValidationProblem((*loc, *found.loc), found.message, found.kind) for found in problems + ) + return (value if entity is None else entity), located + _CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { BloscCodec.identifier: BloscCodec, BytesCodec.identifier: BytesCodec, Crc32cCodec.identifier: Crc32cCodec, GzipCodec.identifier: GzipCodec, + ShardingIndexedCodec.identifier: ShardingIndexedCodec, TransposeCodec.identifier: TransposeCodec, } _EXTENSION_CODECS: Final[dict[str, type[MetadataEntity]]] = { + CastValueCodec.identifier: CastValueCodec, ScaleOffsetCodec.identifier: ScaleOffsetCodec, ZstdCodec.identifier: ZstdCodec, } @@ -129,6 +164,7 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] StringDataType.identifier: StringDataType, NumpyDatetime64DataType.identifier: NumpyDatetime64DataType, NumpyTimedelta64DataType.identifier: NumpyTimedelta64DataType, + StructDataType.identifier: StructDataType, } _CORE_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = { diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 656a509ed7..0daac40d94 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -4,7 +4,25 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ -from typing import Final, Literal, NotRequired +from collections.abc import Mapping +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + DATA_TYPE, + CodecKind, + Coerced, + Loc, + MemberTypes, + MetadataEntity, + is_json_value, + one_of, + problem, +) + +if TYPE_CHECKING: + from zarr_metadata.v3._registry import Context from typing_extensions import TypedDict @@ -99,8 +117,10 @@ class CastValueCodecObject(TypedDict, closed=True): "CAST_OUT_OF_RANGE_MODE", "CAST_ROUNDING_MODE", "CAST_VALUE_CODEC_NAME", + "SCALAR_MAP_KEYS", "CastOutOfRangeMode", "CastRoundingMode", + "CastValueCodec", "CastValueCodecConfiguration", "CastValueCodecMetadata", "CastValueCodecName", @@ -108,3 +128,97 @@ class CastValueCodecObject(TypedDict, closed=True): "ScalarMap", "ScalarMapEntry", ] + + +SCALAR_MAP_KEYS: Final = ("encode", "decode") +"""The two directions a `scalar_map` can override, both optional.""" + + +def _is_scalar_map(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """An object of `[old, new]` pairs per direction.""" + if not isinstance(value, Mapping): + return problem(loc, f"expected an object, got {value!r}") + mapping = cast("Mapping[str, object]", value) + found: list[ValidationProblem] = [] + for key in mapping: + if key not in SCALAR_MAP_KEYS: + found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) + for key in SCALAR_MAP_KEYS: + if key in mapping: + found.extend(_is_scalar_pairs(mapping[key], (*loc, key))) + return tuple(found) + + +def _is_scalar_pairs(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple): + return problem(loc, f"expected an array of [old, new] pairs, got {value!r}") + entries = cast("tuple[object, ...]", value) + found: list[ValidationProblem] = [] + for index, entry in enumerate(entries): + pair = cast("tuple[object, ...]", entry) if isinstance(entry, tuple) else () + if len(pair) != 2: + found.extend(problem((*loc, index), f"expected an [old, new] pair, got {entry!r}")) + continue + for position, scalar in enumerate(pair): + found.extend(is_json_value(scalar, (*loc, index, position))) + return tuple(found) + + +def _is_data_type_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """A metadata field -- which data type it names is settled on recursion.""" + if not isinstance(value, (str, Mapping)): + return problem(loc, f"expected a data type, got {value!r}") + return () + + +@dataclass(frozen=True) +class CastValueCodec(MetadataEntity): + """The `cast_value` codec, coerced from its metadata. + + Holds the data type it casts to, so like `sharding_indexed` it is + read in a scope rather than on its own. + """ + + data_type: MetadataEntity | object = None + rounding: CastRoundingMode | None = None + out_of_range: CastOutOfRangeMode | None = None + scalar_map: ScalarMap | None = None + + identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME + kind: ClassVar[CodecKind] = "array_array" + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { + "data_type": (True, _is_data_type_field), + "rounding": (False, one_of(CAST_ROUNDING_MODE)), + "out_of_range": (False, one_of(CAST_OUT_OF_RANGE_MODE)), + "scalar_map": (False, _is_scalar_map), + } + + @classmethod + def coerce(cls, value: object, context: "Context") -> Coerced[Self]: + codec, problems = super().coerce(value, context) + if codec is None: + return None, problems + data_type, found = context.coerce(DATA_TYPE, codec.data_type, ("data_type",)) + return replace(codec, data_type=data_type), (*problems, *found) + + def problems(self) -> tuple[ValidationProblem, ...]: + """Whatever the data type being cast to says about itself.""" + if not isinstance(self.data_type, MetadataEntity): + return () + return tuple( + ValidationProblem(("data_type", *entry.loc), entry.message, entry.kind) + for entry in self.data_type.problems() + ) + + def configuration(self) -> dict[str, object]: + """The target data type in its canonical spelling.""" + members = super().configuration() + data_type = self.data_type + if isinstance(data_type, MetadataEntity): + members["data_type"] = data_type.to_json() + return members + + def to_json(self) -> CastValueCodecObject: + return cast("CastValueCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 0ce466b5cf..76bc2b4bc0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -4,7 +4,26 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html """ -from typing import Final, Literal, NotRequired +from collections.abc import Mapping +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + CODECS, + CodecKind, + Coerced, + Loc, + MemberTypes, + MetadataEntity, + is_int, + one_of, + problem, + sequence_of, +) + +if TYPE_CHECKING: + from zarr_metadata.v3._registry import Context from typing_extensions import TypedDict @@ -69,8 +88,111 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): "SHARDING_INDEXED_CODEC_NAME", "SHARDING_INDEX_LOCATION", "ShardingIndexLocation", + "ShardingIndexedCodec", "ShardingIndexedCodecConfiguration", "ShardingIndexedCodecMetadata", "ShardingIndexedCodecName", "ShardingIndexedCodecObject", ] + + +def _is_field_tuple(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """An array of metadata fields -- their names are checked on recursion.""" + if not isinstance(value, tuple): + return problem(loc, f"expected an array of codecs, got {value!r}") + entries = cast("tuple[object, ...]", value) + return tuple( + found + for index, entry in enumerate(entries) + if not isinstance(entry, (str, Mapping)) + for found in problem((*loc, index), f"expected a metadata field, got {entry!r}") + ) + + +def _coerce_pipeline( + entries: tuple[object, ...], context: "Context", loc: Loc +) -> tuple[tuple[MetadataEntity | object, ...], tuple[ValidationProblem, ...]]: + """Every entry of one pipeline, read in `context`.""" + coerced: list[MetadataEntity | object] = [] + problems: list[ValidationProblem] = [] + for index, entry in enumerate(entries): + codec, found = context.coerce(CODECS, entry, (*loc, index)) + coerced.append(codec) + problems.extend(found) + return tuple(coerced), tuple(problems) + + +@dataclass(frozen=True) +class ShardingIndexedCodec(MetadataEntity): + """The `sharding_indexed` codec, coerced from its metadata. + + Holds two codec pipelines, so it is one of the few entities that + needs the scope it is being read in: an entry of either pipeline is + itself an entity, read the same way this one was. + """ + + chunk_shape: tuple[int, ...] = () + codecs: tuple[MetadataEntity | object, ...] = () + index_codecs: tuple[MetadataEntity | object, ...] = () + index_location: ShardingIndexLocation | None = None + + identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME + kind: ClassVar[CodecKind] = "array_bytes" + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = { + "chunk_shape": (True, sequence_of(is_int)), + "codecs": (True, _is_field_tuple), + "index_codecs": (True, _is_field_tuple), + "index_location": (False, one_of(SHARDING_INDEX_LOCATION)), + } + + @classmethod + def coerce(cls, value: object, context: "Context") -> Coerced[Self]: + shard, problems = super().coerce(value, context) + if shard is None: + return None, problems + inner, from_inner = _coerce_pipeline(shard.codecs, context, ("codecs",)) + index, from_index = _coerce_pipeline(shard.index_codecs, context, ("index_codecs",)) + return ( + replace(shard, codecs=inner, index_codecs=index), + (*problems, *from_inner, *from_index), + ) + + def problems(self) -> tuple[ValidationProblem, ...]: + """This shard's own values, and those of the codecs it holds. + + Whether the two pipelines are well *formed* -- one array-to-bytes + codec, in the right order -- spans the whole chain, so the rules + layer asks that. + """ + found: list[ValidationProblem] = [ + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(self.chunk_shape) + if extent < 1 + ] + for member in ("codecs", "index_codecs"): + for position, codec in enumerate(cast("tuple[object, ...]", getattr(self, member))): + if isinstance(codec, MetadataEntity): + found.extend( + ValidationProblem((member, position, *entry.loc), entry.message, entry.kind) + for entry in codec.problems() + ) + return tuple(found) + + def configuration(self) -> dict[str, object]: + """The two pipelines in their canonical spelling, entry by entry.""" + members = super().configuration() + for member in ("codecs", "index_codecs"): + members[member] = tuple( + entry.to_json() if isinstance(entry, MetadataEntity) else entry + for entry in cast("tuple[object, ...]", members[member]) + ) + return members + + def to_json(self) -> ShardingIndexedCodecObject: + return cast("ShardingIndexedCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index f5d1168b82..7577c8a2e0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass BOOL_DATA_TYPE_NAME: Final = "bool" """The `data_type` value for the `bool` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class BoolDataType(MetadataEntity): +class BoolDataType(DataTypeEntity): """The `bool` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = BOOL_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 09040de036..6f6e4f20ed 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass BYTES_DATA_TYPE_NAME: Final = "bytes" """The `data_type` value for the variable-length `bytes` type.""" @@ -53,7 +53,8 @@ def base64_bytes(value: str) -> Base64Bytes: @dataclass(frozen=True) -class BytesDataType(MetadataEntity): +class BytesDataType(DataTypeEntity): """The `bytes` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "variable_length" identifier: ClassVar[str] = BYTES_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index e800efd960..e6f71b9789 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass from zarr_metadata.v3.data_type.float64 import Float64FillValue COMPLEX128_DATA_TYPE_NAME: Final = "complex128" @@ -41,7 +41,8 @@ @dataclass(frozen=True) -class Complex128DataType(MetadataEntity): +class Complex128DataType(DataTypeEntity): """The `complex128` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = COMPLEX128_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 3d1b8d42c7..367b919fbf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass from zarr_metadata.v3.data_type.float32 import Float32FillValue COMPLEX64_DATA_TYPE_NAME: Final = "complex64" @@ -41,7 +41,8 @@ @dataclass(frozen=True) -class Complex64DataType(MetadataEntity): +class Complex64DataType(DataTypeEntity): """The `complex64` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = COMPLEX64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 970b0aa536..102253f7ae 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass FLOAT16_DATA_TYPE_NAME: Final = "float16" """The `data_type` value for the `float16` type.""" @@ -79,7 +79,8 @@ def hex_float16(value: str) -> HexFloat16: @dataclass(frozen=True) -class Float16DataType(MetadataEntity): +class Float16DataType(DataTypeEntity): """The `float16` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = FLOAT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 594dcfce0a..5a086a9cb1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass FLOAT32_DATA_TYPE_NAME: Final = "float32" """The `data_type` value for the `float32` type.""" @@ -79,7 +79,8 @@ def hex_float32(value: str) -> HexFloat32: @dataclass(frozen=True) -class Float32DataType(MetadataEntity): +class Float32DataType(DataTypeEntity): """The `float32` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = FLOAT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index dda2be3a63..1c8fd99769 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass FLOAT64_DATA_TYPE_NAME: Final = "float64" """The `data_type` value for the `float64` type.""" @@ -80,7 +80,8 @@ def hex_float64(value: str) -> HexFloat64: @dataclass(frozen=True) -class Float64DataType(MetadataEntity): +class Float64DataType(DataTypeEntity): """The `float64` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = FLOAT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index 145573f27d..30cffe724c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass INT16_DATA_TYPE_NAME: Final = "int16" """The `data_type` value for the `int16` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Int16DataType(MetadataEntity): +class Int16DataType(DataTypeEntity): """The `int16` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = INT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 3a27176d4f..4a99aff4b3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass INT32_DATA_TYPE_NAME: Final = "int32" """The `data_type` value for the `int32` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Int32DataType(MetadataEntity): +class Int32DataType(DataTypeEntity): """The `int32` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = INT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index 293da17f7b..e17c661f87 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass INT64_DATA_TYPE_NAME: Final = "int64" """The `data_type` value for the `int64` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Int64DataType(MetadataEntity): +class Int64DataType(DataTypeEntity): """The `int64` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = INT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 875e775750..80b7f43018 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass INT8_DATA_TYPE_NAME: Final = "int8" """The `data_type` value for the `int8` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Int8DataType(MetadataEntity): +class Int8DataType(DataTypeEntity): """The `int8` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = INT8_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 1c8364328f..f3fa99ebe8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -11,8 +11,9 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + DataTypeEntity, MemberTypes, - MetadataEntity, + StorageClass, is_int, one_of, problem, @@ -77,12 +78,13 @@ class NumpyDatetime64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyDatetime64DataType(MetadataEntity): +class NumpyDatetime64DataType(DataTypeEntity): """The `numpy.datetime64` data type, coerced from its metadata.""" unit: NumpyTimeUnit = "generic" scale_factor: int = 1 + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME configuration_required: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 55a57aea45..374f19434b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -11,8 +11,9 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + DataTypeEntity, MemberTypes, - MetadataEntity, + StorageClass, is_int, one_of, problem, @@ -97,12 +98,13 @@ class NumpyTimedelta64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyTimedelta64DataType(MetadataEntity): +class NumpyTimedelta64DataType(DataTypeEntity): """The `numpy.timedelta64` data type, coerced from its metadata.""" unit: NumpyTimeUnit = "generic" scale_factor: int = 1 + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME configuration_required: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 5f35bbba21..2de2cdd401 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -16,7 +16,8 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( Coerced, - MetadataEntity, + DataTypeEntity, + StorageClass, named_configuration, problem, ) @@ -78,7 +79,7 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: @dataclass(frozen=True) -class RawBytesDataType(MetadataEntity): +class RawBytesDataType(DataTypeEntity): """An `r` raw-bytes data type, coerced from its metadata. One class for the whole family, because `r8` and `r4096` differ only @@ -93,6 +94,7 @@ class RawBytesDataType(MetadataEntity): data_type_name: str = "r8" + scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = RAW_BYTES_FAMILY @classmethod diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 04fdccd3e6..6c9d34b02f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass STRING_DATA_TYPE_NAME: Final = "string" """The `data_type` value for the `string` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class StringDataType(MetadataEntity): +class StringDataType(DataTypeEntity): """The `string` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "variable_length" identifier: ClassVar[str] = STRING_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 87a437951d..3ccad54d44 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -5,7 +5,23 @@ """ from collections.abc import Mapping -from typing import Final, Literal, NotRequired +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + DATA_TYPE, + Coerced, + DataTypeEntity, + Loc, + MemberTypes, + MetadataEntity, + StorageClass, + problem, +) + +if TYPE_CHECKING: + from zarr_metadata.v3._registry import Context from typing_extensions import ReadOnly, TypedDict @@ -19,6 +35,10 @@ """Literal type of the `name` field of the `struct` data type.""" +STRUCT_FIELD_KEYS: Final = ("name", "data_type") +"""The members a struct field entry carries, both required.""" + + class StructField(TypedDict, closed=True): """ A single field entry inside a structured dtype. @@ -59,9 +79,171 @@ class Struct(TypedDict, closed=True): __all__ = [ "STRUCT_DATA_TYPE_NAME", + "STRUCT_FIELD_KEYS", "Struct", "StructConfiguration", + "StructDataType", "StructDataTypeName", "StructField", + "StructFieldComponent", "StructFillValue", ] + + +def _is_fields(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """An array of `{name, data_type}` objects. + + Whether a `data_type` names anything is settled on recursion; this + only asks whether the field entry has the two members at all. + """ + if not isinstance(value, tuple): + return problem(loc, f"expected an array of struct fields, got {value!r}") + entries = cast("tuple[object, ...]", value) + found: list[ValidationProblem] = [] + for index, entry in enumerate(entries): + if not isinstance(entry, Mapping): + found.extend(problem((*loc, index), f"expected a struct field, got {entry!r}")) + continue + field = cast("Mapping[str, object]", entry) + for key in STRUCT_FIELD_KEYS: + if key not in field: + found.extend(problem((*loc, index), f"missing required key {key!r}", "missing_key")) + for key in field: + if key not in STRUCT_FIELD_KEYS: + found.extend(problem((*loc, index), f"unexpected key {key!r}", "unknown_key")) + if "name" in field and not isinstance(field["name"], str): + found.extend( + problem((*loc, index, "name"), f"expected a string, got {field['name']!r}") + ) + return tuple(found) + + +@dataclass(frozen=True) +class StructFieldComponent: + """One field of a struct: a name, and the type of its values. + + `data_type` is the coerced entity when the field's type is in scope, + and the metadata untouched when it is not. + """ + + name: str + data_type: MetadataEntity | object + + def to_json(self) -> StructField: + data_type = self.data_type + return cast( + "StructField", + { + "name": self.name, + "data_type": ( + data_type.to_json() if isinstance(data_type, MetadataEntity) else data_type + ), + }, + ) + + +@dataclass(frozen=True) +class StructDataType(DataTypeEntity): + """The `struct` data type, coerced from its metadata. + + A record of named fields, each with a data type of its own -- so this + is a data type that contains data types, and needs the scope it is + read in to make sense of them. + """ + + fields: tuple[StructFieldComponent, ...] = () + + identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME + scalar_storage: ClassVar[StorageClass] = "single_byte" + + configuration_required: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = {"fields": (True, _is_fields)} + + @classmethod + def coerce(cls, value: object, context: "Context") -> Coerced[Self]: + struct, problems = super().coerce(value, context) + if struct is None: + return None, problems + fields: list[StructFieldComponent] = [] + found: list[ValidationProblem] = [] + for index, entry in enumerate(cast("tuple[object, ...]", struct.fields)): + field = cast("Mapping[str, object]", entry) + data_type, from_field = context.coerce( + DATA_TYPE, field["data_type"], ("fields", index, "data_type") + ) + found.extend(from_field) + fields.append( + StructFieldComponent(name=cast("str", field["name"]), data_type=data_type) + ) + return replace(struct, fields=tuple(fields)), (*problems, *found) + + def storage_class(self) -> StorageClass | None: + """The widest class among the fields. + + A struct of `uint8` and `int32` is `multi_byte`, and one holding + a `string` anywhere inside it is `variable_length`. None when any + field's type is out of scope: the answer would be a guess. + """ + widest: StorageClass = "single_byte" + for field in self.fields: + if not isinstance(field.data_type, DataTypeEntity): + return None + found = field.data_type.storage_class() + if found is None: + return None + if found == "variable_length": + return "variable_length" + if found == "multi_byte": + widest = "multi_byte" + return widest + + def problems(self) -> tuple[ValidationProblem, ...]: + """What a struct can judge about its own fields. + + Names have to exist, be non-empty and be distinct, because a fill + value addresses fields by name. Field types have to be fixed-size, + because a record's layout is otherwise not determined. + """ + found: list[ValidationProblem] = [] + if len(self.fields) == 0: + found.extend( + problem(("fields",), "expected at least one struct field", "invalid_value") + ) + seen: dict[str, int] = {} + for index, field in enumerate(self.fields): + at: Loc = ("fields", index) + if field.name == "": + found.extend( + problem((*at, "name"), "expected a non-empty field name", "invalid_value") + ) + first = seen.setdefault(field.name, index) + if first != index: + found.extend( + problem( + (*at, "name"), + f"duplicate field name {field.name!r}, already used by field {first}", + "invalid_value", + ) + ) + if not isinstance(field.data_type, DataTypeEntity): + continue + if field.data_type.storage_class() == "variable_length": + found.extend( + problem( + (*at, "data_type"), + "struct fields must use fixed-size data types", + "invalid_value", + ) + ) + found.extend( + ValidationProblem((*at, "data_type", *entry.loc), entry.message, entry.kind) + for entry in field.data_type.problems() + ) + return tuple(found) + + def configuration(self) -> dict[str, object]: + """Each field in its canonical spelling, type included.""" + return {"fields": tuple(field.to_json() for field in self.fields)} + + def to_json(self) -> Struct: + return cast("Struct", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index aa2b5dc07c..cb87b4b29a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass UINT16_DATA_TYPE_NAME: Final = "uint16" """The `data_type` value for the `uint16` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Uint16DataType(MetadataEntity): +class Uint16DataType(DataTypeEntity): """The `uint16` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = UINT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index fded4f2ab6..9cb4eb6861 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass UINT32_DATA_TYPE_NAME: Final = "uint32" """The `data_type` value for the `uint32` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Uint32DataType(MetadataEntity): +class Uint32DataType(DataTypeEntity): """The `uint32` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = UINT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index e06a8db7c4..6ec1f92985 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass UINT64_DATA_TYPE_NAME: Final = "uint64" """The `data_type` value for the `uint64` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Uint64DataType(MetadataEntity): +class Uint64DataType(DataTypeEntity): """The `uint64` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = UINT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index 7a1f6a3e85..d8d3ae916b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._entity import DataTypeEntity, StorageClass UINT8_DATA_TYPE_NAME: Final = "uint8" """The `data_type` value for the `uint8` type.""" @@ -28,7 +28,8 @@ @dataclass(frozen=True) -class Uint8DataType(MetadataEntity): +class Uint8DataType(DataTypeEntity): """The `uint8` data type. The name says everything.""" + scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = UINT8_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index d2e0025bc4..ae7b04a774 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -30,9 +30,14 @@ ) from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration from zarr_metadata.v3.codec.bytes import BytesCodec, BytesCodecConfiguration +from zarr_metadata.v3.codec.cast_value import CastValueCodec, CastValueCodecConfiguration from zarr_metadata.v3.codec.crc32c import Crc32cCodec, Empty from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecConfiguration from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec, ScaleOffsetCodecConfiguration +from zarr_metadata.v3.codec.sharding_indexed import ( + ShardingIndexedCodec, + ShardingIndexedCodecConfiguration, +) from zarr_metadata.v3.codec.transpose import TransposeCodec, TransposeCodecConfiguration from zarr_metadata.v3.codec.zstd import ZstdCodec, ZstdCodecConfiguration from zarr_metadata.v3.data_type.bool import BoolDataType @@ -56,6 +61,7 @@ ) from zarr_metadata.v3.data_type.raw import RawBytesDataType from zarr_metadata.v3.data_type.string import StringDataType +from zarr_metadata.v3.data_type.struct import StructConfiguration, StructDataType from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType @@ -71,9 +77,11 @@ CONFIGURATIONS: dict[str, tuple[type[MetadataEntity], type | None]] = { "codecs:blosc": (BloscCodec, BloscCodecConfiguration), "codecs:bytes": (BytesCodec, BytesCodecConfiguration), + "codecs:cast_value": (CastValueCodec, CastValueCodecConfiguration), "codecs:crc32c": (Crc32cCodec, Empty), "codecs:gzip": (GzipCodec, GzipCodecConfiguration), "codecs:scale_offset": (ScaleOffsetCodec, ScaleOffsetCodecConfiguration), + "codecs:sharding_indexed": (ShardingIndexedCodec, ShardingIndexedCodecConfiguration), "codecs:transpose": (TransposeCodec, TransposeCodecConfiguration), "codecs:zstd": (ZstdCodec, ZstdCodecConfiguration), "chunk_grid:regular": (RegularChunkGrid, RegularChunkGridConfiguration), @@ -99,6 +107,7 @@ "data_type:complex64": (Complex64DataType, None), "data_type:complex128": (Complex128DataType, None), "data_type:bytes": (BytesDataType, None), + "data_type:struct": (StructDataType, StructConfiguration), "data_type:string": (StringDataType, None), # The one exception. `r` is a family, so the class holds the # spelling that picks a member of it -- a field with no configuration @@ -188,3 +197,18 @@ def test_an_entity_round_trips_through_its_json_form() -> None: assert problems == () assert codec is not None assert codec.to_json() == original + + +def test_an_unknown_key_is_reported_without_losing_the_member() -> None: + # The document is invalid either way, but dropping the member would + # make the entity describe something the document does not say. + codec, problems = CastValueCodec.coerce( + { + "name": "cast_value", + "configuration": {"data_type": "int8", "scalar_map": {"encode": (), "enc": ()}}, + }, + CORE_AND_EXTENSIONS, + ) + assert [problem.kind for problem in problems] == ["unknown_key"] + assert codec is not None + assert codec.scalar_map == {"encode": (), "enc": ()} From c48cec2a55a1ac1d62e6270dd7cd341b8c074874 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:09:48 +0200 Subject: [PATCH 029/107] feat(zarr-metadata): a data type says what it accepts as a fill value `_check_fill_for_dtype` was a chain of name comparisons ending in a table lookup, with the struct case recursing by hand. It becomes a method: `DataTypeEntity.fill_value_problems`, defaulting to accepting anything, because a type this package does not model is not ours to judge. The families are where the widths go. Every integer type differs from every other only in its bounds, every float only in its hex parser, every complex only in its component -- so each family is written once and the type supplies the number. `struct` recurses through its fields by calling the same method on each, which is the hand-rolled recursion turned back into a method call. `must_understand` becomes keyword-only. It belongs to the envelope, not the configuration, and it was silently taking the first positional slot of every entity -- `RawBytesDataType("r16")` set the flag and left the name at its default. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 17 +- .../zarr_metadata/v3/data_type/_families.py | 153 ++++++++++++++++++ .../src/zarr_metadata/v3/data_type/bool.py | 13 +- .../src/zarr_metadata/v3/data_type/bytes.py | 21 ++- .../zarr_metadata/v3/data_type/complex128.py | 8 +- .../zarr_metadata/v3/data_type/complex64.py | 8 +- .../src/zarr_metadata/v3/data_type/float16.py | 7 +- .../src/zarr_metadata/v3/data_type/float32.py | 7 +- .../src/zarr_metadata/v3/data_type/float64.py | 7 +- .../src/zarr_metadata/v3/data_type/int16.py | 6 +- .../src/zarr_metadata/v3/data_type/int32.py | 6 +- .../src/zarr_metadata/v3/data_type/int64.py | 6 +- .../src/zarr_metadata/v3/data_type/int8.py | 6 +- .../v3/data_type/numpy_datetime64.py | 4 +- .../v3/data_type/numpy_timedelta64.py | 4 +- .../src/zarr_metadata/v3/data_type/raw.py | 14 ++ .../src/zarr_metadata/v3/data_type/string.py | 13 +- .../src/zarr_metadata/v3/data_type/struct.py | 35 ++++ .../src/zarr_metadata/v3/data_type/uint16.py | 6 +- .../src/zarr_metadata/v3/data_type/uint32.py | 6 +- .../src/zarr_metadata/v3/data_type/uint64.py | 6 +- .../src/zarr_metadata/v3/data_type/uint8.py | 6 +- .../tests/v3/test_fill_values.py | 97 +++++++++++ 23 files changed, 419 insertions(+), 37 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py create mode 100644 packages/zarr-metadata/tests/v3/test_fill_values.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 3b684e1003..007065ac3d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -39,7 +39,7 @@ from __future__ import annotations from collections.abc import Mapping as _Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast from typing_extensions import TypeIs @@ -247,7 +247,11 @@ class MetadataEntity: rather than a constant, a member another member renders meaningless. """ - must_understand: bool = True + # Keyword-only: it is the envelope's member, not the configuration's, + # and it would otherwise take the first positional slot of every + # entity -- so `RawBytesDataType("r16")` would set this instead of + # the field it reads as. + must_understand: bool = field(default=True, kw_only=True) identifier: ClassVar[str] """The name this entity is registered under. @@ -368,6 +372,15 @@ def storage_class(self) -> StorageClass | None: """ return type(self).scalar_storage + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + """Why `value` is not a fill value of this type, if it is not. + + Default: nothing. A data type this package does not model accepts + whatever its extension says it does, and guessing would reject + valid documents. + """ + return () + def named_configuration( value: object, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py new file mode 100644 index 0000000000..fdf04a5357 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -0,0 +1,153 @@ +"""What each family of data types accepts as a fill value. + +A fill value is judged against the data type, which makes it a question +the data type answers: `DataTypeEntity.fill_value_problems`. The families +here exist because the answer is the same for every width in a family +apart from one number -- the range, the hex parser, the component type -- +so each family is written once and parameterised by that number. + +The alternative, a table keyed by data type name, is what this replaces: +it put the knowledge of what `int32` accepts somewhere other than +`int32`, and needed a drift test to keep the two in step. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Final + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + DataTypeEntity, + StorageClass, + is_integer, + problem, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from zarr_metadata.v3._entity import Loc + +FLOAT_SPECIALS: Final = ("NaN", "Infinity", "-Infinity") +"""The three non-finite floats the spec spells as strings.""" + + +def as_sequence(value: object) -> tuple[object, ...] | None: + """`value` as a tuple if it is a JSON array, else None. + + A string is a sequence in Python and never a JSON array, so it is + excluded. + """ + if isinstance(value, str) or not isinstance(value, Sequence): + return None + return tuple(value) # type: ignore[arg-type] + + +def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[ValidationProblem, ...]: + """An array of `expected` integers in [0, 255], or any length if None.""" + items = as_sequence(value) + if items is None: + return problem(loc, f"expected an array of byte values, got {value!r}", "invalid_value") + if expected is not None and len(items) != expected: + return problem(loc, f"expected {expected} byte values, got {len(items)}", "invalid_value") + return tuple( + found + for index, item in enumerate(items) + if not (is_integer(item) and 0 <= item <= 255) + for found in problem( + (*loc, index), f"expected integers in [0, 255], got {item!r}", "invalid_value" + ) + ) + + +@dataclass(frozen=True) +class IntegerDataType(DataTypeEntity): + """A fixed-width integer. The width is the whole difference.""" + + bounds: ClassVar[tuple[int, int]] + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + low, high = type(self).bounds + if not is_integer(value): + return problem(loc, f"expected an integer, got {value!r}", "invalid_value") + if not low <= value <= high: + return problem( + loc, f"expected an integer in [{low}, {high}], got {value!r}", "invalid_value" + ) + return () + + +@dataclass(frozen=True) +class FloatDataType(DataTypeEntity): + """A binary float. A fill value may be a number, a named non-finite, or hex.""" + + scalar_storage: ClassVar[StorageClass] = "multi_byte" + hex_parser: ClassVar[Callable[[str], object]] + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + if is_integer(value) or isinstance(value, float): + return () + if not isinstance(value, str): + return problem(loc, f"expected a number or string, got {value!r}", "invalid_value") + if value in FLOAT_SPECIALS: + return () + try: + type(self).hex_parser(value) + except ValueError: + return problem( + loc, + f"expected a number, one of 'NaN'/'Infinity'/'-Infinity', or a " + f"{type(self).identifier} hex string, got {value!r}", + "invalid_value", + ) + return () + + +@dataclass(frozen=True) +class ComplexDataType(DataTypeEntity): + """A complex number: a `[real, imag]` pair of the component float type.""" + + scalar_storage: ClassVar[StorageClass] = "multi_byte" + component: ClassVar[type[FloatDataType]] + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + pair = as_sequence(value) + if pair is None or len(pair) != 2: + return problem(loc, f"expected a [real, imag] pair, got {value!r}", "invalid_value") + component = type(self).component() + return tuple( + ValidationProblem(found.loc, f"invalid component: {found.message}", found.kind) + for index, part in enumerate(pair) + for found in component.fill_value_problems(part, (*loc, index)) + ) + + +@dataclass(frozen=True) +class NumpyTimeDataType(DataTypeEntity): + """A numpy time scalar: a signed 64-bit count of units, or `NaT`.""" + + scalar_storage: ClassVar[StorageClass] = "multi_byte" + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + if value == "NaT": + return () + if not is_integer(value): + return problem( + loc, f"expected a signed 64-bit integer or 'NaT', got {value!r}", "invalid_value" + ) + if not -(2**63) <= value <= 2**63 - 1: + return problem(loc, f"expected a signed 64-bit integer, got {value!r}", "invalid_value") + return () + + +__all__ = [ + "FLOAT_SPECIALS", + "ComplexDataType", + "FloatDataType", + "IntegerDataType", + "NumpyTimeDataType", + "as_sequence", + "byte_values", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index 7577c8a2e0..d240ffdf2d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -7,7 +7,13 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + DataTypeEntity, + Loc, + StorageClass, + problem, +) BOOL_DATA_TYPE_NAME: Final = "bool" """The `data_type` value for the `bool` type.""" @@ -33,3 +39,8 @@ class BoolDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = BOOL_DATA_TYPE_NAME + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + if not isinstance(value, bool): + return problem(loc, f"expected a boolean, got {value!r}", "invalid_value") + return () diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 6f6e4f20ed..c52e5237e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -8,7 +8,14 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + DataTypeEntity, + Loc, + StorageClass, + problem, +) +from zarr_metadata.v3.data_type._families import byte_values BYTES_DATA_TYPE_NAME: Final = "bytes" """The `data_type` value for the variable-length `bytes` type.""" @@ -58,3 +65,15 @@ class BytesDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "variable_length" identifier: ClassVar[str] = BYTES_DATA_TYPE_NAME + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + """Base64, or an array of byte values of any length.""" + if isinstance(value, str): + try: + base64_bytes(value) + except ValueError: + return problem( + loc, f"expected standard-alphabet base64, got {value!r}", "invalid_value" + ) + return () + return byte_values(value, None, loc) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index e6f71b9789..12ff217385 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -7,8 +7,9 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass -from zarr_metadata.v3.data_type.float64 import Float64FillValue +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import ComplexDataType, FloatDataType +from zarr_metadata.v3.data_type.float64 import Float64DataType, Float64FillValue COMPLEX128_DATA_TYPE_NAME: Final = "complex128" """The `data_type` value for the `complex128` type.""" @@ -41,8 +42,9 @@ @dataclass(frozen=True) -class Complex128DataType(DataTypeEntity): +class Complex128DataType(ComplexDataType): """The `complex128` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + component: ClassVar[type[FloatDataType]] = Float64DataType identifier: ClassVar[str] = COMPLEX128_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 367b919fbf..cf06a9214d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -7,8 +7,9 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass -from zarr_metadata.v3.data_type.float32 import Float32FillValue +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import ComplexDataType, FloatDataType +from zarr_metadata.v3.data_type.float32 import Float32DataType, Float32FillValue COMPLEX64_DATA_TYPE_NAME: Final = "complex64" """The `data_type` value for the `complex64` type.""" @@ -41,8 +42,9 @@ @dataclass(frozen=True) -class Complex64DataType(DataTypeEntity): +class Complex64DataType(ComplexDataType): """The `complex64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + component: ClassVar[type[FloatDataType]] = Float32DataType identifier: ClassVar[str] = COMPLEX64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 102253f7ae..42121410b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -5,10 +5,12 @@ """ import re +from collections.abc import Callable from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import FloatDataType FLOAT16_DATA_TYPE_NAME: Final = "float16" """The `data_type` value for the `float16` type.""" @@ -79,8 +81,9 @@ def hex_float16(value: str) -> HexFloat16: @dataclass(frozen=True) -class Float16DataType(DataTypeEntity): +class Float16DataType(FloatDataType): """The `float16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float16) identifier: ClassVar[str] = FLOAT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 5a086a9cb1..4c589a052a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -5,10 +5,12 @@ """ import re +from collections.abc import Callable from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import FloatDataType FLOAT32_DATA_TYPE_NAME: Final = "float32" """The `data_type` value for the `float32` type.""" @@ -79,8 +81,9 @@ def hex_float32(value: str) -> HexFloat32: @dataclass(frozen=True) -class Float32DataType(DataTypeEntity): +class Float32DataType(FloatDataType): """The `float32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float32) identifier: ClassVar[str] = FLOAT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 1c8fd99769..3b24fc999c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -5,10 +5,12 @@ """ import re +from collections.abc import Callable from dataclasses import dataclass from typing import ClassVar, Final, Literal, NewType -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import FloatDataType FLOAT64_DATA_TYPE_NAME: Final = "float64" """The `data_type` value for the `float64` type.""" @@ -80,8 +82,9 @@ def hex_float64(value: str) -> HexFloat64: @dataclass(frozen=True) -class Float64DataType(DataTypeEntity): +class Float64DataType(FloatDataType): """The `float64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float64) identifier: ClassVar[str] = FLOAT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index 30cffe724c..fb295e3dfc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType INT16_DATA_TYPE_NAME: Final = "int16" """The `data_type` value for the `int16` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Int16DataType(DataTypeEntity): +class Int16DataType(IntegerDataType): """The `int16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + bounds: ClassVar[tuple[int, int]] = (-32768, 32767) identifier: ClassVar[str] = INT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 4a99aff4b3..7d35f033cd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType INT32_DATA_TYPE_NAME: Final = "int32" """The `data_type` value for the `int32` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Int32DataType(DataTypeEntity): +class Int32DataType(IntegerDataType): """The `int32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + bounds: ClassVar[tuple[int, int]] = (-2147483648, 2147483647) identifier: ClassVar[str] = INT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index e17c661f87..365370a9dc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType INT64_DATA_TYPE_NAME: Final = "int64" """The `data_type` value for the `int64` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Int64DataType(DataTypeEntity): +class Int64DataType(IntegerDataType): """The `int64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + bounds: ClassVar[tuple[int, int]] = (-9223372036854775808, 9223372036854775807) identifier: ClassVar[str] = INT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 80b7f43018..5a2bf185ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType INT8_DATA_TYPE_NAME: Final = "int8" """The `data_type` value for the `int8` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Int8DataType(DataTypeEntity): +class Int8DataType(IntegerDataType): """The `int8` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" + bounds: ClassVar[tuple[int, int]] = (-128, 127) identifier: ClassVar[str] = INT8_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index f3fa99ebe8..204fd99c9d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -11,13 +11,13 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( - DataTypeEntity, MemberTypes, StorageClass, is_int, one_of, problem, ) +from zarr_metadata.v3.data_type._families import NumpyTimeDataType from zarr_metadata.v3.data_type.numpy_timedelta64 import ( NUMPY_TIME_MAX_SCALE_FACTOR, NUMPY_TIME_UNIT, @@ -78,7 +78,7 @@ class NumpyDatetime64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyDatetime64DataType(DataTypeEntity): +class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" unit: NumpyTimeUnit = "generic" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 374f19434b..739f576c32 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -11,13 +11,13 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( - DataTypeEntity, MemberTypes, StorageClass, is_int, one_of, problem, ) +from zarr_metadata.v3.data_type._families import NumpyTimeDataType NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" """The `name` field value of the `numpy.timedelta64` data type.""" @@ -98,7 +98,7 @@ class NumpyTimedelta64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyTimedelta64DataType(DataTypeEntity): +class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" unit: NumpyTimeUnit = "generic" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 2de2cdd401..6085ded208 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -17,10 +17,12 @@ from zarr_metadata.v3._entity import ( Coerced, DataTypeEntity, + Loc, StorageClass, named_configuration, problem, ) +from zarr_metadata.v3.data_type._families import byte_values RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) """A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). @@ -135,3 +137,15 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: "ZarrV3MetadataFieldJSON", {"name": self.data_type_name, "must_understand": False}, ) + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + """One byte value per byte of the scalar. + + A malformed name says nothing about how wide the scalar is, so + there is no length to check against; `problems` reports the name. + """ + try: + raw_bytes_dtype_name(self.data_type_name) + except ValueError: + return () + return byte_values(value, int(self.data_type_name[1:]) // 8, loc) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 6c9d34b02f..826273607c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -7,7 +7,13 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import ( + DataTypeEntity, + Loc, + StorageClass, + problem, +) STRING_DATA_TYPE_NAME: Final = "string" """The `data_type` value for the `string` type.""" @@ -33,3 +39,8 @@ class StringDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "variable_length" identifier: ClassVar[str] = STRING_DATA_TYPE_NAME + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + if not isinstance(value, str): + return problem(loc, f"expected a string, got {value!r}", "invalid_value") + return () diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 3ccad54d44..a4821d4fd7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -241,6 +241,41 @@ def problems(self) -> tuple[ValidationProblem, ...]: ) return tuple(found) + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + """A fill value per field, addressed by name. + + Every field needs one and nothing else may appear, because a + record's value is not determined otherwise. A field whose type is + out of scope still needs an entry -- that much is structural -- + but what the entry holds is left unjudged. + """ + if not isinstance(value, Mapping): + return problem( + loc, + f"expected an object of per-field fill values, got {value!r}", + "invalid_value", + ) + fills = cast("Mapping[str, object]", value) + found: list[ValidationProblem] = [] + for field in self.fields: + at: Loc = (*loc, field.name) + if field.name not in fills: + found.extend( + problem( + at, f"missing fill value for struct field {field.name!r}", "missing_key" + ) + ) + continue + if not isinstance(field.data_type, DataTypeEntity): + continue + found.extend(field.data_type.fill_value_problems(fills[field.name], at)) + declared = {field.name for field in self.fields} + found.extend( + ValidationProblem((*loc, key), f"unknown struct fill field {key!r}", "unknown_key") + for key in sorted(fills.keys() - declared) + ) + return tuple(found) + def configuration(self) -> dict[str, object]: """Each field in its canonical spelling, type included.""" return {"fields": tuple(field.to_json() for field in self.fields)} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index cb87b4b29a..d87340123c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType UINT16_DATA_TYPE_NAME: Final = "uint16" """The `data_type` value for the `uint16` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Uint16DataType(DataTypeEntity): +class Uint16DataType(IntegerDataType): """The `uint16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + bounds: ClassVar[tuple[int, int]] = (0, 65535) identifier: ClassVar[str] = UINT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index 9cb4eb6861..ab0c7b2cce 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType UINT32_DATA_TYPE_NAME: Final = "uint32" """The `data_type` value for the `uint32` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Uint32DataType(DataTypeEntity): +class Uint32DataType(IntegerDataType): """The `uint32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + bounds: ClassVar[tuple[int, int]] = (0, 4294967295) identifier: ClassVar[str] = UINT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 6ec1f92985..241f4eb819 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType UINT64_DATA_TYPE_NAME: Final = "uint64" """The `data_type` value for the `uint64` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Uint64DataType(DataTypeEntity): +class Uint64DataType(IntegerDataType): """The `uint64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" + bounds: ClassVar[tuple[int, int]] = (0, 18446744073709551615) identifier: ClassVar[str] = UINT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index d8d3ae916b..fb5fabbf5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -7,7 +7,8 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal -from zarr_metadata.v3._entity import DataTypeEntity, StorageClass +from zarr_metadata.v3._entity import StorageClass +from zarr_metadata.v3.data_type._families import IntegerDataType UINT8_DATA_TYPE_NAME: Final = "uint8" """The `data_type` value for the `uint8` type.""" @@ -28,8 +29,9 @@ @dataclass(frozen=True) -class Uint8DataType(DataTypeEntity): +class Uint8DataType(IntegerDataType): """The `uint8` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" + bounds: ClassVar[tuple[int, int]] = (0, 255) identifier: ClassVar[str] = UINT8_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py new file mode 100644 index 0000000000..5eb45b5fa2 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -0,0 +1,97 @@ +"""What each data type accepts as a fill value. + +The knowledge used to live in a table keyed by data type name; now each +type answers for itself, so these exercise the families directly rather +than through a document. +""" + +from __future__ import annotations + +import pytest + +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS + +# (data type metadata, a fill value it accepts) +ACCEPTED: dict[str, tuple[object, object]] = { + "bool": ("bool", True), + "int8-low": ("int8", -128), + "uint64-high": ("uint64", 2**64 - 1), + "float64-number": ("float64", 1.5), + "float64-integer": ("float64", 1), + "float32-special": ("float32", "NaN"), + "float16-hex": ("float16", "0x3e00"), + "complex64-pair": ("complex64", (1.0, "-Infinity")), + "string": ("string", "hello"), + "bytes-base64": ("bytes", "aGk="), + "bytes-array": ("bytes", (1, 2, 3)), + "raw-exact-width": ("r16", (0, 255)), + "raw-malformed-unjudged": ("r12", "anything"), + "time-integer": ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, + -1, + ), + "time-nat": ( + {"name": "numpy.timedelta64", "configuration": {"unit": "s", "scale_factor": 1}}, + "NaT", + ), + "struct": ( + {"name": "struct", "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}}, + {"a": 7}, + ), +} + +# (data type metadata, a fill value it rejects, part of the reason) +REJECTED: dict[str, tuple[object, object, str]] = { + "bool-integer": ("bool", 1, "expected a boolean"), + "int8-above-range": ("int8", 128, "[-128, 127]"), + "uint8-negative": ("uint8", -1, "[0, 255]"), + "float64-not-a-number": ("float64", "nope", "hex string"), + "float64-boolean": ("float64", True, "expected a number or string"), + "complex64-single": ("complex64", 1.0, "[real, imag] pair"), + "complex64-bad-part": ("complex64", (1.0, "nope"), "invalid component"), + "string-integer": ("string", 3, "expected a string"), + "bytes-bad-base64": ("bytes", "!!", "standard-alphabet base64"), + "raw-wrong-width": ("r16", (1, 2, 3), "expected 2 byte values"), + "raw-byte-out-of-range": ("r8", (256,), "[0, 255]"), + "time-not-integer": ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, + 1.5, + "signed 64-bit integer or 'NaT'", + ), + "struct-missing-field": ( + {"name": "struct", "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}}, + {}, + "missing fill value", + ), + "struct-unknown-field": ( + {"name": "struct", "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}}, + {"a": 1, "b": 2}, + "unknown struct fill field", + ), +} + + +def _data_type(metadata: object) -> object: + name = metadata if isinstance(metadata, str) else metadata["name"] # type: ignore[index] + entity_type = CORE_AND_EXTENSIONS.resolve("data_type", name) # type: ignore[arg-type] + assert entity_type is not None, metadata + entity, problems = entity_type.coerce(metadata, CORE_AND_EXTENSIONS) + assert problems == (), problems + return entity + + +@pytest.mark.parametrize(("metadata", "fill"), ACCEPTED.values(), ids=list(ACCEPTED)) +def test_accepts(metadata: object, fill: object) -> None: + assert _data_type(metadata).fill_value_problems(fill) == () # type: ignore[attr-defined] + + +@pytest.mark.parametrize(("metadata", "fill", "reason"), REJECTED.values(), ids=list(REJECTED)) +def test_error_rejects(metadata: object, fill: object, reason: str) -> None: + problems = _data_type(metadata).fill_value_problems(fill) # type: ignore[attr-defined] + assert problems, f"expected {fill!r} to be rejected" + assert any(reason in problem.message for problem in problems), problems + + +def test_an_unmodelled_data_type_judges_nothing() -> None: + # Extension openness: a fill value we cannot interpret is not wrong. + assert CORE_AND_EXTENSIONS.resolve("data_type", "mycorp.decimal") is None From edbbeda1e77f8d095b61a743f296a79403ae3edb Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:14:10 +0200 Subject: [PATCH 030/107] feat(zarr-metadata): a codec transforms the array; a grid divides it Two more name-keyed tables become methods. `ChunkGrid.of` dispatched on the grid's name to read its configuration; now each grid entity builds its own, and a grid that cannot be read falls back to the rank the array pins. `_TRANSITIONS` was a registry of "what this codec does to the array it receives", keyed by name and populated by a decorator; now it is `CodecEntity.transition`, defaulting to None so a modelled codec that forgets to say still fails closed. `ChunkGrid`, `ArrayParts` and the rest move from `rules` to `v3._parts`, because a codec cannot own its transition while the thing it transforms lives in the layer above it. Coercion also now normalizes a tuple's contents, not just a list's. Raw JSON arrives as lists all the way down, but a hand-written tuple holding a list was left half-normalized and failed its own type check. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 44 +++- .../src/zarr_metadata/v3/_parts.py | 238 ++++++++++++++++++ .../v3/chunk_grid/rectilinear.py | 34 ++- .../zarr_metadata/v3/chunk_grid/regular.py | 9 +- .../src/zarr_metadata/v3/codec/blosc.py | 4 +- .../src/zarr_metadata/v3/codec/bytes.py | 4 +- .../src/zarr_metadata/v3/codec/cast_value.py | 15 +- .../src/zarr_metadata/v3/codec/crc32c.py | 4 +- .../src/zarr_metadata/v3/codec/gzip.py | 4 +- .../zarr_metadata/v3/codec/scale_offset.py | 13 +- .../v3/codec/sharding_indexed.py | 3 +- .../src/zarr_metadata/v3/codec/transpose.py | 14 +- .../src/zarr_metadata/v3/codec/zstd.py | 4 +- 13 files changed, 368 insertions(+), 22 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_parts.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 007065ac3d..ae6255f5c2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -45,6 +45,7 @@ from typing_extensions import TypeIs from zarr_metadata.model._validation import ValidationProblem, is_json +from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: from collections.abc import Callable, Mapping @@ -52,6 +53,7 @@ from zarr_metadata.model._validation import ProblemKind from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._parts import ArrayParts from zarr_metadata.v3._registry import Context EntityT = TypeVar("EntityT", bound="MetadataEntity") @@ -182,8 +184,9 @@ def _as_tuples(value: object) -> object: own type says tuple -- and two documents differing only in that would compare unequal. """ - if isinstance(value, list): - return tuple(_as_tuples(entry) for entry in cast("list[object]", value)) + if isinstance(value, (list, tuple)): + entries = cast("list[object] | tuple[object, ...]", value) + return tuple(_as_tuples(entry) for entry in entries) if isinstance(value, _Mapping): entries = cast("Mapping[str, object]", value) return {key: _as_tuples(entry) for key, entry in entries.items()} @@ -352,6 +355,41 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: return cast("ZarrV3MetadataFieldJSON", entry) +@dataclass(frozen=True) +class CodecEntity(MetadataEntity): + """An entity that occupies a position in the codec pipeline.""" + + kind: ClassVar[CodecKind] + + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + """What the next codec in the chain sees, or None if undeterminable. + + Only an array-to-array codec has anything to say: the two later + kinds end shape propagation by construction, one by consuming the + array and the other by never having had it. + + The default is None, so a modelled codec that forgets to say how + it transforms the array stops propagation rather than silently + claiming to leave it alone. Failing closed here costs a judgment; + failing open would invent one. + """ + return None + + +@dataclass(frozen=True) +class ChunkGridEntity(MetadataEntity): + """An entity that divides an array into the parts a pipeline encodes.""" + + def grid(self, array_shape: object) -> ChunkGrid: + """What this grid divides an array of `array_shape` into. + + The array shape is a parameter because neither determines a grid + alone: a grid whose own metadata cannot be read still has the + array's rank, and rank is enough for several rules. + """ + return ChunkGrid.unreadable(array_shape) + + @dataclass(frozen=True) class DataTypeEntity(MetadataEntity): """An entity that says how the array's scalars are stored. @@ -415,6 +453,8 @@ def named_configuration( "CHUNK_KEY_ENCODING", "CODECS", "DATA_TYPE", + "ChunkGridEntity", + "CodecEntity", "CodecKind", "Coerced", "DataTypeEntity", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py new file mode 100644 index 0000000000..30bb780b84 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py @@ -0,0 +1,238 @@ +"""How an array is divided, and what a codec chain does to it. + +A codec pipeline encodes one chunk, but the same pipeline encodes *every* +chunk, so a rule about a pipeline is a statement about all of them at +once: a shard's inner chunk shape must divide every chunk it will ever be +handed, not some representative one. `ChunkGrid` is what makes that +statement expressible, and `ArrayParts` is what one codec hands the next. + +Three pieces of metadata divide an array: a document's `chunk_grid`, a +`sharding_indexed` codec's `chunk_shape` (a regular grid over the chunk +that codec receives), and that codec's shard index, whose shape the spec +derives from the other two. Each grid entity builds its own, because what +a grid divides an array into is the grid's own knowledge. + +Per dimension, and plurally +--------------------------- +`extents` holds one entry per dimension: the set of lengths that +dimension's chunks take. A regular grid gives singletons; a rectilinear +grid gives `{30, 34}` on an axis whose chunks differ; `None` marks an axis +this package cannot read. `rank` survives even when no extent does, +because every chunk of an array has the array's rank whatever divides it. + +Collapsing any of that loses real judgments. A rectilinear grid uniform on +one axis still pins that axis, and a shard is judged there while declining +on the others. + +Prior art +--------- +zarrs builds its grid from metadata *and* the array shape +(`ChunkGrid::create(metadata, array_shape)`) because neither determines a +grid alone, keeps `dimensionality()` total rather than optional, and +reports `chunk_edge_lengths(dimension)` per dimension for the reason +above. Its codec chain distinguishes "no global grid, but one per chunk" +(`ChunkGridMapped::ChunkLocal`) from "nothing known" (`::None`); an +`extents` entry of `None` beside a known `rank` is the per-dimension form +of that distinction. + +- https://github.com/zarrs/zarrs/blob/main/zarrs_chunk_grid/src/lib.rs +- https://github.com/zarrs/zarrs/blob/main/zarrs_codec/src/lib.rs +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, TypeAlias, cast + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + + +Extents: TypeAlias = "tuple[frozenset[int] | None, ...]" +"""One entry per dimension: the lengths that dimension's chunks take. + +A singleton is a uniform axis. `None` is an axis whose lengths this +package cannot determine — distinct from an empty set, which would claim +the axis has no chunks at all. +""" + + +def _positive_int(value: object) -> int | None: + """`value` as a chunk length, or None if it is not a usable one.""" + if isinstance(value, bool) or not isinstance(value, int): + return None + return value if value >= 1 else None + + +def _rank_of(array_shape: object) -> int | None: + """The number of dimensions `array_shape` declares, if it declares any.""" + if not isinstance(array_shape, tuple): + return None + dimensions = cast("tuple[object, ...]", array_shape) + if not all(isinstance(v, int) and not isinstance(v, bool) for v in dimensions): + return None + return len(dimensions) + + +def _uniform(lengths: Sequence[object]) -> Extents: + """Extents for a grid whose chunks are the same everywhere.""" + return tuple( + None if (length := _positive_int(value)) is None else frozenset({length}) + for value in lengths + ) + + +def _rectilinear_axis(spec: object) -> frozenset[int] | None: + """The lengths one rectilinear dimension's chunks take. + + A bare integer is a regular step, so every chunk is that long. An + explicit list names them, with `[size, count]` pairs standing for + repeats; the distinct sizes are what any divisibility question needs. + """ + step = _positive_int(spec) + if step is not None: + return frozenset({step}) + if not isinstance(spec, tuple): + return None + lengths: set[int] = set() + for item in cast("tuple[object, ...]", spec): + size = _positive_int(item) + if size is None and isinstance(item, tuple): + pair = cast("tuple[object, ...]", item) + if len(pair) != 2 or _positive_int(pair[1]) is None: + return None + size = _positive_int(pair[0]) + if size is None: + return None + lengths.add(size) + return frozenset(lengths) if len(lengths) != 0 else None + + +@dataclass(frozen=True, slots=True) +class ChunkGrid: + """The division of an array into the parts a codec pipeline encodes. + + `metadata` is the grid as the document spells it, kept so that a rule + for a grid this package does not model can still read its own + configuration. It is absent for a grid this package derived rather + than read — the regular grid a sharding codec imposes, or a transposed + grid — so nothing may validate it or report a location into it. + """ + + rank: int | None + extents: Extents | None + metadata: ZarrV3MetadataFieldJSON | None = None + + @classmethod + def unreadable(cls, array_shape: object) -> ChunkGrid: + """A grid nothing is known about but the rank the array pins. + + Every third-party grid, and every modelled one whose own metadata + could not be read: the array still has a rank, and a rule about + rank is still answerable. + """ + rank = _rank_of(array_shape) + return cls(rank, None if rank is None else (None,) * rank) + + @classmethod + def derived(cls, extents: Extents) -> ChunkGrid: + """A grid this package computed rather than read from a document.""" + return cls(len(extents), extents) + + @classmethod + def regular(cls, lengths: Sequence[object]) -> ChunkGrid: + """The regular grid a sharding codec's `chunk_shape` imposes.""" + return cls.derived(_uniform(lengths)) + + def permuted(self, order: Sequence[int]) -> ChunkGrid: + """This grid with its dimensions reordered by `order`. + + A transposed grid is still a grid — permuting a regular one gives + a regular one — but it is no longer the grid the document wrote, + so the metadata does not survive the trip. + + Declines on anything that is not a permutation of this grid's rank. + The caller checks that too and reports it, but an order is only + shape-validated as a tuple of integers, so this must not be the + thing that decides whether a validator raises `IndexError`. + """ + if self.extents is None or sorted(order) != list(range(len(self.extents))): + return ChunkGrid(self.rank, None) + return ChunkGrid.derived(tuple(self.extents[axis] for axis in order)) + + def axis(self, dimension: int) -> frozenset[int] | None: + """The lengths `dimension`'s chunks take, or None if undetermined.""" + if self.extents is None or dimension >= len(self.extents): + return None + return self.extents[dimension] + + +UNKNOWN_GRID: ChunkGrid = ChunkGrid(None, None) +"""A grid nothing is known about — not even how many dimensions it has.""" + + +def shard_index_grid(shard: ChunkGrid, inner: Sequence[object]) -> ChunkGrid: + """The grid of a shard's index array. + + The spec derives it from the two shapes around it: "The index is an + array with 64-bit unsigned integers with a shape that matches the + chunks per shard tuple with an appended dimension of size 2." The + index is one array rather than a divided one, so each axis holds a + single length — except that under a rectilinear grid the shard itself + varies, so the chunk count varies with it and the axis holds every + value it takes. + """ + inner_extents = _uniform(inner) + trailing: frozenset[int] | None = frozenset({2}) + if shard.extents is None or len(shard.extents) != len(inner_extents): + return ChunkGrid.derived((*(None,) * len(inner_extents), trailing)) + counts: list[frozenset[int] | None] = [] + for lengths, divisor in zip(shard.extents, inner_extents, strict=True): + if lengths is None or divisor is None: + counts.append(None) + continue + step = next(iter(divisor)) + quotients = {length // step for length in lengths if length % step == 0} + counts.append(frozenset(quotients) if len(quotients) == len(lengths) else None) + return ChunkGrid.derived((*counts, trailing)) + + +@dataclass(frozen=True, slots=True) +class ArrayParts: + """Every part of an array a codec will be handed, and their type. + + The parts an array is divided into, not the fields of its metadata. + Plural deliberately: one pipeline encodes every chunk, so a rule about + it quantifies over all of them — a shard's inner chunk shape must + divide *every* chunk, which under a rectilinear grid is several + different lengths. + + `data_type` is the metadata-field value verbatim, because rules compare + it by name, and it is `None` where the element type is undetermined + while the array itself is not. That happens inside a shard: the inner + grid is the sharding codec's own `chunk_shape` whatever reached it, so + an unreadable codec upstream costs the type and not the parts. `None` + in place of the whole value means something else again — that there is + no array here at all, past the array->bytes boundary or beyond a codec + that could have changed anything. + """ + + grid: ChunkGrid + data_type: ZarrV3MetadataFieldJSON | None + + def with_grid(self, grid: ChunkGrid) -> ArrayParts: + return replace(self, grid=grid) + + def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArrayParts: + return replace(self, data_type=data_type) + + +__all__ = [ + "UNKNOWN_GRID", + "ArrayParts", + "ChunkGrid", + "Extents", + "shard_index_grid", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index bfa6f30617..105290e28a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -11,13 +11,14 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + ChunkGridEntity, Loc, MemberTypes, - MetadataEntity, is_integer, one_of, problem, ) +from zarr_metadata.v3._parts import ChunkGrid RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" """The `name` field value of the rectilinear chunk grid.""" @@ -155,8 +156,28 @@ def _is_dim_specs(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return tuple(found) +def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: + """The lengths one dimension's chunks take, or None if undetermined. + + A bare integer is a regular step, so every chunk is that long. An + explicit list names them, with `[size, count]` pairs standing for + repeats; the distinct sizes are what any divisibility question needs. + None for a non-positive length, which `problems` reports -- a grid + that does not tile answers nothing about what divides it. + """ + if isinstance(spec, int): + return frozenset({spec}) if spec > 0 else None + lengths: set[int] = set() + for item in spec: + size = item if isinstance(item, int) else item[0] + if size < 1 or (not isinstance(item, int) and item[1] < 1): + return None + lengths.add(size) + return frozenset(lengths) if len(lengths) != 0 else None + + @dataclass(frozen=True) -class RectilinearChunkGrid(MetadataEntity): +class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" kind: Literal["inline"] = "inline" @@ -207,6 +228,15 @@ def problems(self) -> tuple[ValidationProblem, ...]: ) return tuple(found) + def grid(self, array_shape: object) -> ChunkGrid: + """The distinct lengths each axis's chunks take. + + Plural per axis, which is the point of a rectilinear grid: an + axis of `[30, 34]` gives `{30, 34}`, and anything asking about + divisibility has to hold for both. + """ + return ChunkGrid.derived(tuple(_axis_lengths(spec) for spec in self.chunk_shapes)) + def configuration(self) -> dict[str, object]: """Run-length encoded, which is the spelling that does not grow. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index fb1a172bf5..b42dcd645c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -11,11 +11,12 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + ChunkGridEntity, MemberTypes, - MetadataEntity, is_int, sequence_of, ) +from zarr_metadata.v3._parts import ChunkGrid REGULAR_CHUNK_GRID_NAME: Final = "regular" """The `name` field value of the regular chunk grid.""" @@ -58,7 +59,7 @@ class RegularChunkGridObject(TypedDict, closed=True): @dataclass(frozen=True) -class RegularChunkGrid(MetadataEntity): +class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" chunk_shape: tuple[int, ...] = () @@ -86,5 +87,9 @@ def problems(self) -> tuple[ValidationProblem, ...]: if extent < 1 ) + def grid(self, array_shape: object) -> ChunkGrid: + """One extent per axis, the same for every chunk on that axis.""" + return ChunkGrid.regular(self.chunk_shape) + def to_json(self) -> RegularChunkGridObject: return cast("RegularChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index a6b657ff9e..e4d7aead06 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -12,8 +12,8 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + CodecEntity, MemberTypes, - MetadataEntity, is_int, one_of, problem, @@ -107,7 +107,7 @@ def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, @dataclass(frozen=True) -class BloscCodec(MetadataEntity): +class BloscCodec(CodecEntity): """The `blosc` codec, coerced from its metadata. Everything blosc knows about itself: the shape its metadata takes, the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index f06c434d24..265ce0c893 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -10,9 +10,9 @@ from typing_extensions import TypedDict from zarr_metadata.v3._entity import ( + CodecEntity, CodecKind, MemberTypes, - MetadataEntity, one_of, ) @@ -77,7 +77,7 @@ class BytesCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BytesCodec(MetadataEntity): +class BytesCodec(CodecEntity): """The `bytes` codec, coerced from its metadata. `endian` is optional and absent means something: a one-byte data type diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 0daac40d94..3d76adedbd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -11,6 +11,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( DATA_TYPE, + CodecEntity, CodecKind, Coerced, Loc, @@ -20,8 +21,10 @@ one_of, problem, ) +from zarr_metadata.v3._parts import ArrayParts if TYPE_CHECKING: + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._registry import Context from typing_extensions import TypedDict @@ -172,7 +175,7 @@ def _is_data_type_field(value: object, loc: Loc) -> tuple[ValidationProblem, ... @dataclass(frozen=True) -class CastValueCodec(MetadataEntity): +class CastValueCodec(CodecEntity): """The `cast_value` codec, coerced from its metadata. Holds the data type it casts to, so like `sharding_indexed` it is @@ -220,5 +223,15 @@ def configuration(self) -> dict[str, object]: members["data_type"] = data_type.to_json() return members + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + """The same parts, holding the type this codec casts to.""" + data_type = self.data_type + return incoming.with_data_type( + cast( + "ZarrV3MetadataFieldJSON", + data_type.to_json() if isinstance(data_type, MetadataEntity) else data_type, + ) + ) + def to_json(self) -> CastValueCodecObject: return cast("CastValueCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 37cd77157c..96ee88be08 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -13,8 +13,8 @@ from typing_extensions import TypedDict from zarr_metadata.v3._entity import ( + CodecEntity, CodecKind, - MetadataEntity, ) CRC32C_CODEC_NAME: Final = "crc32c" @@ -61,7 +61,7 @@ class Crc32cCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class Crc32cCodec(MetadataEntity): +class Crc32cCodec(CodecEntity): """The `crc32c` codec, coerced from its metadata. The name says everything: a checksum has nothing to configure. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 0d602d2e04..073c801f65 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -11,9 +11,9 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + CodecEntity, CodecKind, MemberTypes, - MetadataEntity, is_int, problem, ) @@ -69,7 +69,7 @@ class GzipCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class GzipCodec(MetadataEntity): +class GzipCodec(CodecEntity): """The `gzip` codec, coerced from its metadata.""" level: int = 5 diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index ea5b12a2f8..f2d4b9b4ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -11,11 +11,12 @@ from zarr_metadata._common import JSONValue from zarr_metadata.v3._entity import ( + CodecEntity, CodecKind, MemberTypes, - MetadataEntity, is_json_value, ) +from zarr_metadata.v3._parts import ArrayParts SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" """The `name` field value of the `scale_offset` codec.""" @@ -72,7 +73,7 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ScaleOffsetCodec(MetadataEntity): +class ScaleOffsetCodec(CodecEntity): """The `scale_offset` codec, coerced from its metadata. Both members are optional and any JSON scalar is well-typed here; what @@ -91,5 +92,13 @@ class ScaleOffsetCodec(MetadataEntity): "scale": (False, is_json_value), } + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + """The same array, element for element. + + The registry entry removed the `astype` field, so this codec no + longer changes the element type -- only the values. + """ + return incoming + def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 76bc2b4bc0..01fe929bfd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -11,6 +11,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CODECS, + CodecEntity, CodecKind, Coerced, Loc, @@ -123,7 +124,7 @@ def _coerce_pipeline( @dataclass(frozen=True) -class ShardingIndexedCodec(MetadataEntity): +class ShardingIndexedCodec(CodecEntity): """The `sharding_indexed` codec, coerced from its metadata. Holds two codec pipelines, so it is one of the few entities that diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 2deccbfa4e..555843ce77 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -11,13 +11,14 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + CodecEntity, CodecKind, MemberTypes, - MetadataEntity, is_int, problem, sequence_of, ) +from zarr_metadata.v3._parts import ArrayParts TRANSPOSE_CODEC_NAME: Final = "transpose" """The `name` field value of the `transpose` codec.""" @@ -65,7 +66,7 @@ class TransposeCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class TransposeCodec(MetadataEntity): +class TransposeCodec(CodecEntity): """The `transpose` codec, coerced from its metadata.""" order: tuple[int, ...] = () @@ -90,5 +91,14 @@ def problems(self) -> tuple[ValidationProblem, ...]: ) return () + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + """The same array with its axes reordered. + + A transposed regular grid is still a regular grid, so the parts + survive the trip; the grid metadata does not, because it is no + longer the grid the document wrote. + """ + return incoming.with_grid(incoming.grid.permuted(self.order)) + def to_json(self) -> TransposeCodecObject: return cast("TransposeCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 7db38b8842..f59808be0e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -13,9 +13,9 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + CodecEntity, CodecKind, MemberTypes, - MetadataEntity, is_bool, is_int, problem, @@ -77,7 +77,7 @@ class ZstdCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ZstdCodec(MetadataEntity): +class ZstdCodec(CodecEntity): """The `zstd` codec, coerced from its metadata.""" level: int = 0 From f4aa34401fe497681b6e5a5fda1257c194eb2d7c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:15:24 +0200 Subject: [PATCH 031/107] fix(zarr-metadata): narrow the codec kind and drop the grid reader the entities replaced Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_parts.py | 26 ------------------- .../src/zarr_metadata/v3/codec/blosc.py | 3 ++- .../src/zarr_metadata/v3/codec/cast_value.py | 10 +++---- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py index 30bb780b84..ab17a44c2f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py @@ -84,32 +84,6 @@ def _uniform(lengths: Sequence[object]) -> Extents: ) -def _rectilinear_axis(spec: object) -> frozenset[int] | None: - """The lengths one rectilinear dimension's chunks take. - - A bare integer is a regular step, so every chunk is that long. An - explicit list names them, with `[size, count]` pairs standing for - repeats; the distinct sizes are what any divisibility question needs. - """ - step = _positive_int(spec) - if step is not None: - return frozenset({step}) - if not isinstance(spec, tuple): - return None - lengths: set[int] = set() - for item in cast("tuple[object, ...]", spec): - size = _positive_int(item) - if size is None and isinstance(item, tuple): - pair = cast("tuple[object, ...]", item) - if len(pair) != 2 or _positive_int(pair[1]) is None: - return None - size = _positive_int(pair[0]) - if size is None: - return None - lengths.add(size) - return frozenset(lengths) if len(lengths) != 0 else None - - @dataclass(frozen=True, slots=True) class ChunkGrid: """The division of an array into the parts a codec pipeline encodes. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index e4d7aead06..80ed72ec99 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -13,6 +13,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, + CodecKind, MemberTypes, is_int, one_of, @@ -122,7 +123,7 @@ class BloscCodec(CodecEntity): typesize: int | None = None identifier: ClassVar[str] = BLOSC_CODEC_NAME - kind: ClassVar[str] = "bytes_bytes" + kind: ClassVar[CodecKind] = "bytes_bytes" # Every member is required but `typesize`, which only means something # when shuffling; `problems` is where that conditional lives. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 3d76adedbd..fdfccf5b72 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -24,7 +24,6 @@ from zarr_metadata.v3._parts import ArrayParts if TYPE_CHECKING: - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._registry import Context from typing_extensions import TypedDict @@ -226,12 +225,9 @@ def configuration(self) -> dict[str, object]: def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type - return incoming.with_data_type( - cast( - "ZarrV3MetadataFieldJSON", - data_type.to_json() if isinstance(data_type, MetadataEntity) else data_type, - ) - ) + if isinstance(data_type, MetadataEntity): + return incoming.with_data_type(data_type.to_json()) + return incoming.with_data_type(cast(ZarrV3MetadataFieldJSON, data_type)) def to_json(self) -> CastValueCodecObject: return cast("CastValueCodecObject", super().to_json()) From 906c8cc7b3560d8584a555efe7014f60e743b43b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:19:07 +0200 Subject: [PATCH 032/107] feat(zarr-metadata): the composition rules move onto the entities too Every remaining rule was "this entity against some part of the document", which is a method that takes that part. A grid answers `shape_problems(array_shape)`, a codec answers `incoming_problems(incoming)`, and the rule registry that dispatched them by name will have nothing left to dispatch. The chain walk moves to `v3._chain` because `sharding_indexed` holds two pipelines and has to walk them to judge itself -- it cannot reach up into `rules` to do that. Ordering is read off each codec's own `kind` rather than three tuples of names. `ArrayParts` now carries the coerced data type rather than the metadata verbatim, so the `bytes` codec asks it for its storage class instead of classifying its name. `ChunkGrid` loses its `metadata` field. It was there so a rule for an unmodelled grid could read its own configuration, and an unmodelled grid no longer reaches a rule at all. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_chain.py | 116 ++++++++++++++++++ .../src/zarr_metadata/v3/_entity.py | 17 +++ .../src/zarr_metadata/v3/_parts.py | 21 ++-- .../v3/chunk_grid/rectilinear.py | 58 ++++++++- .../zarr_metadata/v3/chunk_grid/regular.py | 21 +++- .../src/zarr_metadata/v3/codec/bytes.py | 32 +++++ .../src/zarr_metadata/v3/codec/cast_value.py | 5 +- .../src/zarr_metadata/v3/codec/transpose.py | 16 +++ 8 files changed, 270 insertions(+), 16 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_chain.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py new file mode 100644 index 0000000000..eac13818df --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py @@ -0,0 +1,116 @@ +"""Walking a codec pipeline, handing each codec what reaches it. + +The spec orders a pipeline `array->array`* `array->bytes` `bytes->bytes`*, +and each array-to-array codec transforms what the next one sees. So a +codec's configuration is judged against the array that *reaches* it, not +against the document's top-level fields: `transpose` permutes the grid, +`cast_value` changes the element type, and a shard that follows either +one sees the transformed array. + +This is where the walk lives rather than in `zarr_metadata.rules`, +because a `sharding_indexed` codec holds two pipelines of its own and has +to walk them to judge itself. + +Propagation stops -- every later codec receives `None` -- at the +array-to-bytes boundary, where there is no array left, and after any +codec that cannot say what it does to the array: one whose name is out of +scope, or a modelled one with no transition. An unknown codec might +change anything, so declining is the only honest answer. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._entity import CodecEntity + +if TYPE_CHECKING: + from collections.abc import Sequence + + from zarr_metadata.v3._entity import Loc + from zarr_metadata.v3._parts import ArrayParts + +_KIND_RANK = {"array_array": 0, "array_bytes": 1, "bytes_bytes": 2} + + +def _label(codec: object) -> str: + if isinstance(codec, CodecEntity): + return repr(type(codec).identifier) + return repr(codec) + + +def order_problems(codecs: Sequence[object], loc: Loc) -> tuple[ValidationProblem, ...]: + """Whether the pipeline is shaped the way the spec orders it. + + A codec out of scope is skipped: it imposes no ordering constraint, + and it makes the exactly-one-`array->bytes` count inconclusive, + because it might be the pipeline's own `array->bytes` stage. So that + count is only checked when every codec is in scope. + """ + problems: list[ValidationProblem] = [] + latest = -1 + array_bytes = 0 + for index, codec in enumerate(codecs): + if not isinstance(codec, CodecEntity): + continue + kind = type(codec).kind + rank = _KIND_RANK[kind] + if rank < latest: + problems.append( + ValidationProblem( + (*loc, index), + f"{kind.replace('_', '->')} codec {_label(codec)} may not " + "follow a later-stage codec in the pipeline", + "invalid_value", + ) + ) + latest = max(latest, rank) + if kind == "array_bytes": + array_bytes += 1 + if array_bytes > 1: + problems.append( + ValidationProblem( + (*loc, index), + f"extra array->bytes codec {_label(codec)}: a pipeline has exactly one", + "invalid_value", + ) + ) + if array_bytes == 0 and all(isinstance(codec, CodecEntity) for codec in codecs): + problems.append( + ValidationProblem(loc, "codec pipeline has no array->bytes codec", "invalid_value") + ) + return tuple(problems) + + +def chain_problems( + codecs: Sequence[object], start: ArrayParts | None, loc: Loc +) -> tuple[ValidationProblem, ...]: + """Every problem this pipeline has, ordering and per-codec alike. + + `start` is what the first codec receives: the document's own array, or + a shard's inner chunk, or its index. + """ + problems = list(order_problems(codecs, loc)) + incoming = start + for index, codec in enumerate(codecs): + if not isinstance(codec, CodecEntity): + # Out of scope: unjudged, and everything after it is too. + incoming = None + continue + problems.extend( + ValidationProblem((*loc, index, *found.loc), found.message, found.kind) + for found in codec.incoming_problems(incoming) + ) + incoming = ( + None + if incoming is None or type(codec).kind != "array_array" + else codec.transition(incoming) + ) + return tuple(problems) + + +__all__ = [ + "chain_problems", + "order_problems", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index ae6255f5c2..8a13d5b26e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -361,6 +361,15 @@ class CodecEntity(MetadataEntity): kind: ClassVar[CodecKind] + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + """Why this codec cannot be applied to the array that reaches it. + + `incoming` is None once the chain can no longer say what reaches + here, and the default answer to that is nothing: declining beats + guessing. Locations are relative to this codec's entry. + """ + return () + def transition(self, incoming: ArrayParts) -> ArrayParts | None: """What the next codec in the chain sees, or None if undeterminable. @@ -380,6 +389,14 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: class ChunkGridEntity(MetadataEntity): """An entity that divides an array into the parts a pipeline encodes.""" + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: + """Why this grid does not divide an array of `array_shape`. + + Locations are relative to the grid's `configuration`. Default: + nothing, for a grid this package reads but has no such rule for. + """ + return () + def grid(self, array_shape: object) -> ChunkGrid: """What this grid divides an array of `array_shape` into. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py index ab17a44c2f..10f8dcf2b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py @@ -47,7 +47,7 @@ if TYPE_CHECKING: from collections.abc import Sequence - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._entity import DataTypeEntity Extents: TypeAlias = "tuple[frozenset[int] | None, ...]" @@ -88,16 +88,14 @@ def _uniform(lengths: Sequence[object]) -> Extents: class ChunkGrid: """The division of an array into the parts a codec pipeline encodes. - `metadata` is the grid as the document spells it, kept so that a rule - for a grid this package does not model can still read its own - configuration. It is absent for a grid this package derived rather - than read — the regular grid a sharding codec imposes, or a transposed - grid — so nothing may validate it or report a location into it. + Nothing here is the metadata: a grid entity keeps its own, and what + reaches a codec is the division, not the spelling of it. A derived + grid -- the regular one a sharding codec imposes, or a transposed one + -- has no metadata to keep anyway. """ rank: int | None extents: Extents | None - metadata: ZarrV3MetadataFieldJSON | None = None @classmethod def unreadable(cls, array_shape: object) -> ChunkGrid: @@ -183,8 +181,9 @@ class ArrayParts: divide *every* chunk, which under a rectilinear grid is several different lengths. - `data_type` is the metadata-field value verbatim, because rules compare - it by name, and it is `None` where the element type is undetermined + `data_type` is the coerced data type, so a rule asks it what it is + rather than comparing names, and it is `None` where the element type + is undetermined while the array itself is not. That happens inside a shard: the inner grid is the sharding codec's own `chunk_shape` whatever reached it, so an unreadable codec upstream costs the type and not the parts. `None` @@ -194,12 +193,12 @@ class ArrayParts: """ grid: ChunkGrid - data_type: ZarrV3MetadataFieldJSON | None + data_type: DataTypeEntity | None def with_grid(self, grid: ChunkGrid) -> ArrayParts: return replace(self, grid=grid) - def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArrayParts: + def with_data_type(self, data_type: DataTypeEntity | None) -> ArrayParts: return replace(self, data_type=data_type) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 105290e28a..2333a47a59 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict @@ -20,6 +20,10 @@ ) from zarr_metadata.v3._parts import ChunkGrid +if TYPE_CHECKING: + from collections.abc import Sequence + + RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" """The `name` field value of the rectilinear chunk grid.""" @@ -156,6 +160,26 @@ def _is_dim_specs(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return tuple(found) +def _covered_extent(spec: tuple[int | tuple[int, int], ...]) -> int | None: + """How much of a dimension an explicit spec covers, or None. + + None when any entry is non-positive: `problems` reports that, and a + total computed from a nonsense entry would be nonsense too. + """ + total = 0 + for item in spec: + if isinstance(item, int): + if item < 1: + return None + total += item + continue + size, count = item + if size < 1 or count < 1: + return None + total += size * count + return total + + def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: """The lengths one dimension's chunks take, or None if undetermined. @@ -228,6 +252,38 @@ def problems(self) -> tuple[ValidationProblem, ...]: ) return tuple(found) + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: + """One spec per dimension, and explicit specs must cover it. + + A bare integer is uniform shorthand, so it covers whatever the + dimension turns out to be and imposes no sum; an explicit list + names every chunk, so the names have to add up. + """ + if not isinstance(array_shape, (list, tuple)): + return () + extents = tuple(cast("Sequence[object]", array_shape)) + if len(self.chunk_shapes) != len(extents): + return problem( + ("chunk_shapes",), + f"chunk_shapes has {len(self.chunk_shapes)} entries but shape has " + f"{len(extents)} dimensions", + "invalid_value", + ) + found: list[ValidationProblem] = [] + for dim, (spec, extent) in enumerate(zip(self.chunk_shapes, extents, strict=True)): + if isinstance(spec, int) or not is_integer(extent): + continue + total = _covered_extent(spec) + if total is not None and total < extent: + found.extend( + problem( + ("chunk_shapes", dim), + f"chunk sizes sum to {total} but must cover shape[{dim}] extent {extent}", + "invalid_value", + ) + ) + return tuple(found) + def grid(self, array_shape: object) -> ChunkGrid: """The distinct lengths each axis's chunks take. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index b42dcd645c..3525a7d03e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict @@ -14,10 +14,15 @@ ChunkGridEntity, MemberTypes, is_int, + problem, sequence_of, ) from zarr_metadata.v3._parts import ChunkGrid +if TYPE_CHECKING: + from collections.abc import Sequence + + REGULAR_CHUNK_GRID_NAME: Final = "regular" """The `name` field value of the regular chunk grid.""" @@ -87,6 +92,20 @@ def problems(self) -> tuple[ValidationProblem, ...]: if extent < 1 ) + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: + """A regular grid must chunk every array dimension.""" + if not isinstance(array_shape, (list, tuple)): + return () + extents = tuple(cast("Sequence[object]", array_shape)) + if len(self.chunk_shape) == len(extents): + return () + return problem( + ("chunk_shape",), + f"chunk_shape has {len(self.chunk_shape)} entries but shape has " + f"{len(extents)} dimensions", + "invalid_value", + ) + def grid(self, array_shape: object) -> ChunkGrid: """One extent per axis, the same for every chunk on that axis.""" return ChunkGrid.regular(self.chunk_shape) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 265ce0c893..69418c9209 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -9,12 +9,16 @@ from typing_extensions import TypedDict +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, + DataTypeEntity, MemberTypes, one_of, + problem, ) +from zarr_metadata.v3._parts import ArrayParts BYTES_CODEC_NAME: Final = "bytes" """The `name` field value of the `bytes` codec.""" @@ -91,5 +95,33 @@ class BytesCodec(CodecEntity): member_types: ClassVar[MemberTypes] = {"endian": (False, one_of(ENDIANNESS))} + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + """The data type reaching here must have a raw byte representation. + + A variable-length type has no fixed one, so this codec cannot + encode it. A multi-byte one has several orderings, so `endian` is + required -- and the message names the type, because inside a + shard's `index_codecs` the array is the shard index, whose + `uint64` type appears nowhere in the document. + """ + data_type = incoming.data_type if incoming is not None else None + if not isinstance(data_type, DataTypeEntity): + return () + storage = data_type.storage_class() + name = type(data_type).identifier + if storage == "variable_length": + return problem( + (), + f"bytes codec is not compatible with variable-length data_type {name!r}", + "invalid_value", + ) + if storage == "multi_byte" and self.endian is None: + return problem( + ("endian",), + f"endian is required for data type {name!r}, which contains multi-byte values", + "missing_key", + ) + return () + def to_json(self) -> BytesCodecObject | BytesCodecName: return cast("BytesCodecObject | BytesCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index fdfccf5b72..dd3135754b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -14,6 +14,7 @@ CodecEntity, CodecKind, Coerced, + DataTypeEntity, Loc, MemberTypes, MetadataEntity, @@ -225,9 +226,7 @@ def configuration(self) -> dict[str, object]: def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type - if isinstance(data_type, MetadataEntity): - return incoming.with_data_type(data_type.to_json()) - return incoming.with_data_type(cast(ZarrV3MetadataFieldJSON, data_type)) + return incoming.with_data_type(data_type if isinstance(data_type, DataTypeEntity) else None) def to_json(self) -> CastValueCodecObject: return cast("CastValueCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 555843ce77..7fd016b9d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -91,6 +91,22 @@ def problems(self) -> tuple[ValidationProblem, ...]: ) return () + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + """A transpose permutes the array it receives, so ranks must agree. + + Judged against what actually reaches this codec: inside a shard + that is the inner chunk, and after another transpose it is that + transpose's output. + """ + rank = incoming.grid.rank if incoming is not None else None + if rank is None or len(self.order) == rank: + return () + return problem( + ("order",), + f"order has {len(self.order)} entries but the incoming array has {rank} dimensions", + "invalid_value", + ) + def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same array with its axes reordered. From 758b62d3b275d7dc3d1434060bd314c7453bf42e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:26:37 +0200 Subject: [PATCH 033/107] feat(zarr-metadata): validate a v3 array by asking its entities `array_problems_v3` is the document-level check the architecture was for: read every extension point in a scope, ask each entity what is wrong with its own values, then ask the questions that span fields by handing an entity the part of the document it needs. It knows nothing about `blosc` or `int32` or `rectilinear`; a new extension is a class and a registry entry, and this module does not change. Checked against the rule registry it replaces, over the existing Hypothesis strategies: identical problems on valid documents, and identical verdicts everywhere. The two differ only in how much they report about an already-invalid entity -- the old layer judged the values of members it could still read beside one it could not. That precision is kept where it was load-bearing. An optional member that fails its type check falls back to absent, because a bad `index_location` says nothing about whether a shard's pipelines are well formed; a required one stops the entity, because there is no honest reading of a `blosc` whose level is a string. `Context.resolve` now gives the entity the last word via `accepts`. Folding finds a candidate -- every `r` spelling is tabled under one invented identifier -- and the candidate says whether the name is really one of its own. Without that, a document could write the identifier itself. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/rules/_documents.py | 26 ++- .../src/zarr_metadata/v3/_chain.py | 7 +- .../src/zarr_metadata/v3/_document.py | 189 ++++++++++++++++++ .../src/zarr_metadata/v3/_entity.py | 59 ++++-- .../src/zarr_metadata/v3/_registry.py | 11 +- .../src/zarr_metadata/v3/codec/blosc.py | 1 + .../src/zarr_metadata/v3/codec/cast_value.py | 10 +- .../src/zarr_metadata/v3/codec/gzip.py | 1 + .../v3/codec/sharding_indexed.py | 87 +++++++- .../src/zarr_metadata/v3/codec/zstd.py | 1 + .../src/zarr_metadata/v3/data_type/struct.py | 8 +- .../tests/rules/test_canonical.py | 4 +- 12 files changed, 368 insertions(+), 36 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_document.py diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index 8268ce9698..c2bee817c4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -30,8 +30,9 @@ ) from zarr_metadata.rules._engine import run_rules from zarr_metadata.rules._v2_array import ZARR_V2_ARRAY_RULES -from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY_RULES from zarr_metadata.rules._v3_group import ZARR_V3_GROUP_RULES +from zarr_metadata.v3._document import array_problems_v3 +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -46,6 +47,21 @@ _StructuralValidator = Callable[[object], tuple[ValidationProblem, ...]] +def _judged_array_v3(normalized: object) -> tuple[ValidationProblem, ...]: + """Structural and semantic problems in a v3 array document. + + The semantic half is `zarr_metadata.v3` asking each entity about + itself and about the parts of the document it meets; this layer + chooses the scope those questions are asked in. + """ + problems = _validate_structure_v3(normalized) + if isinstance(normalized, Mapping): + problems = problems + array_problems_v3( + cast("Mapping[str, object]", normalized), CORE_AND_EXTENSIONS + ) + return tuple(problems) + + def _judged( normalized: object, structure: _StructuralValidator, rules: Sequence[Rule] ) -> tuple[ValidationProblem, ...]: @@ -63,13 +79,13 @@ def _judged( def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a valid v3 array document. - Structural problems (from the model layer) and composition problems - (from `ZARR_V3_ARRAY_RULES`) are reported together. JSON arrays are + Structural problems (from the model layer) and semantic problems + (from the entities themselves) are reported together. JSON arrays are normalized to tuples before judgment, so list-spelled documents (e.g. fresh `json.loads` output) are judged at the canonical data level rather than rejected for their spelling. """ - return _judged(arrays_to_tuples(value), _validate_structure_v3, ZARR_V3_ARRAY_RULES) + return _judged_array_v3(arrays_to_tuples(value)) def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: @@ -80,7 +96,7 @@ def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: problem found. """ normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_structure_v3, ZARR_V3_ARRAY_RULES) + problems = _judged_array_v3(normalized) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3ArrayMetadataJSON", normalized) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py index eac13818df..08d659dfaa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.v3._entity import CodecEntity +from zarr_metadata.v3._entity import CodecEntity, within if TYPE_CHECKING: from collections.abc import Sequence @@ -98,10 +98,7 @@ def chain_problems( # Out of scope: unjudged, and everything after it is too. incoming = None continue - problems.extend( - ValidationProblem((*loc, index, *found.loc), found.message, found.kind) - for found in codec.incoming_problems(incoming) - ) + problems.extend(within((*loc, index), codec.incoming_problems(incoming))) incoming = ( None if incoming is None or type(codec).kind != "array_array" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py new file mode 100644 index 0000000000..f7aa370fb9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -0,0 +1,189 @@ +"""A whole v3 array document, read as entities and judged as a whole. + +The document-level check is a composition of the entities' own checks, +not a second implementation of them. It does three things in order: + +1. read every extension point in a scope, which is type-space; +2. ask each entity what is wrong with its own values; +3. ask the questions that span fields -- the fill value against the data + type, the grid against the shape, the pipeline against the array -- + each by handing an entity the part of the document it needs. + +Nothing here knows what `blosc` or `int32` or `rectilinear` is. A new +extension is a class and a registry entry, and this module does not +change. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final, cast + +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._chain import chain_problems +from zarr_metadata.v3._entity import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, + ChunkGridEntity, + DataTypeEntity, + ExtensionPointField, + MetadataEntity, + within, +) +from zarr_metadata.v3._parts import ArrayParts, ChunkGrid + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from zarr_metadata.v3._registry import Context + + +@dataclass(frozen=True, slots=True) +class ArrayDocumentV3: + """A v3 array document with its extension points read as entities. + + A field holds the value untouched where its name was out of scope, so + an unmodelled extension survives the reading and is simply not judged. + """ + + document: Mapping[str, object] + data_type: MetadataEntity | object + chunk_grid: MetadataEntity | object + chunk_key_encoding: MetadataEntity | object + codecs: tuple[MetadataEntity | object, ...] + + @property + def parts(self) -> ArrayParts: + """The array the codec pipeline is handed.""" + shape = self.document.get("shape") + # A grid out of scope still divides an array of some rank, and the + # shape is what pins it -- which is enough to catch a shard whose + # inner chunk has the wrong number of dimensions. + grid = ( + self.chunk_grid.grid(shape) + if isinstance(self.chunk_grid, ChunkGridEntity) + else ChunkGrid.unreadable(shape) + ) + return ArrayParts( + grid, self.data_type if isinstance(self.data_type, DataTypeEntity) else None + ) + + +# The three extension points a document names once, and the field each is +# named in. `codecs` is the fourth and holds a list, so it is separate. +_SINGLE_FIELDS: Final[tuple[tuple[ExtensionPointField, str], ...]] = ( + (DATA_TYPE, "data_type"), + (CHUNK_GRID, "chunk_grid"), + (CHUNK_KEY_ENCODING, "chunk_key_encoding"), +) + + +def read_array_v3( + document: Mapping[str, object], context: Context +) -> tuple[ArrayDocumentV3, tuple[ValidationProblem, ...]]: + """`document`'s extension points, read in `context`. + + Type-space only: what comes back is well-typed by construction, and + the problems are the reasons some of it is not an entity. + """ + read: dict[str, MetadataEntity | object] = {} + problems: list[ValidationProblem] = [] + for field, key in _SINGLE_FIELDS: + value = document.get(key) + if value is None: + read[key] = None + continue + entity, found = context.coerce(field, value, (key,)) + read[key] = entity + problems.extend(found) + codecs: list[MetadataEntity | object] = [] + entries = document.get("codecs") + if isinstance(entries, (list, tuple)): + for index, entry in enumerate(cast("Sequence[object]", entries)): + codec, found = context.coerce(CODECS, entry, ("codecs", index)) + codecs.append(codec) + problems.extend(found) + return ( + ArrayDocumentV3( + document=document, + data_type=read["data_type"], + chunk_grid=read["chunk_grid"], + chunk_key_encoding=read["chunk_key_encoding"], + codecs=tuple(codecs), + ), + tuple(problems), + ) + + +def _entity_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: + """What each entity says is wrong with its own values.""" + found: list[ValidationProblem] = [] + for _, key in _SINGLE_FIELDS: + entity = getattr(array, key) + if isinstance(entity, MetadataEntity): + found.extend(within((key,), entity.problems())) + for index, codec in enumerate(array.codecs): + if isinstance(codec, MetadataEntity): + found.extend(within(("codecs", index), codec.problems())) + return tuple(found) + + +def _fill_value_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: + """The fill value, judged by the data type it fills.""" + if not isinstance(array.data_type, DataTypeEntity) or "fill_value" not in array.document: + return () + return array.data_type.fill_value_problems(array.document["fill_value"], ("fill_value",)) + + +def _dimension_names_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: + """One name per dimension, if names are given at all.""" + names = array.document.get("dimension_names") + shape = array.document.get("shape") + if not isinstance(names, (list, tuple)) or not isinstance(shape, (list, tuple)): + return () + given = len(cast("Sequence[object]", names)) + rank = len(cast("Sequence[object]", shape)) + if given == rank: + return () + return ( + ValidationProblem( + ("dimension_names",), + f"dimension_names has {given} entries but shape has {rank} dimensions", + "invalid_value", + ), + ) + + +def _grid_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: + """The chunk grid, judged against the array it divides.""" + if not isinstance(array.chunk_grid, ChunkGridEntity): + return () + return within(("chunk_grid",), array.chunk_grid.shape_problems(array.document.get("shape"))) + + +def array_problems_v3( + document: Mapping[str, object], context: Context +) -> tuple[ValidationProblem, ...]: + """Every semantic problem in `document`, read in `context`. + + Expects a document the model layer has already accepted, so every + member is present and typed as its TypedDict declares. + """ + array, problems = read_array_v3(document, context) + return ( + *problems, + *_entity_problems(array), + *_fill_value_problems(array), + *_grid_problems(array), + *_dimension_names_problems(array), + *chain_problems(array.codecs, array.parts, ("codecs",)), + ) + + +__all__ = [ + "ArrayDocumentV3", + "array_problems_v3", + "read_array_v3", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 8a13d5b26e..40b30db2a9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -48,7 +48,7 @@ from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Mapping, Sequence from typing import Self from zarr_metadata.model._validation import ProblemKind @@ -195,17 +195,25 @@ def _as_tuples(value: object) -> object: def coerce_members( configuration: Mapping[str, object], types: MemberTypes -) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: +) -> tuple[dict[str, object], tuple[ValidationProblem, ...], bool]: """The members `types` declares, taken from `configuration`. - Returns what was accepted and every problem found: a missing required - member, a member of the wrong type, and a key the entity does not - declare. Only a key it does not declare is survivable -- a caller can - report it without abandoning the entity -- so problems are returned - rather than raised and the caller decides. + Returns what was accepted, every problem found, and whether the entity + is still worth building. Three kinds of problem, and they differ in + that last part: + + - a key the entity does not declare says the value carries something + extra, not that it is wrong; + - an *optional* member of the wrong type leaves that member absent, + and everything else about the entity is still readable -- a bad + `index_location` says nothing about whether a shard's pipelines + are well formed, and silencing them would lose a real judgment; + - a *required* member missing or of the wrong type does stop it. + There is no honest reading of a `blosc` whose level is a string. """ problems: list[ValidationProblem] = [] members: dict[str, object] = {} + usable = True for key in configuration: if key not in types: problems.extend(problem(("configuration",), f"unexpected key {key!r}", "unknown_key")) @@ -215,6 +223,7 @@ def coerce_members( problems.extend( problem(("configuration", key), f"missing required key {key!r}", "missing_key") ) + usable = False continue # Normalized before the check, so a check only ever sees the tuples # the TypedDicts declare -- never the lists raw JSON arrives as. @@ -226,7 +235,9 @@ def coerce_members( # dropping it here would make `to_json` lose what was written. if all(entry.kind == "unknown_key" for entry in found): members[key] = value - return members, tuple(problems) + elif required: + usable = False + return members, tuple(problems), usable # No `slots=True`, deliberately: it rebuilds the class, which leaves the @@ -306,10 +317,8 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: "missing_key", ) configuration = cast("Mapping[str, object]", {}) - members, found = coerce_members(configuration, cls.member_types) - # An unknown key is worth reporting but does not stop the entity - # from being read: every member it declares was still understood. - if any(entry.kind != "unknown_key" for entry in found): + members, found, usable = coerce_members(configuration, cls.member_types) + if not usable: return None, found return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] @@ -361,6 +370,13 @@ class CodecEntity(MetadataEntity): kind: ClassVar[CodecKind] + variable_size: ClassVar[bool] = False + """Whether this codec's output size depends on the bytes it is given. + + A compressor's does, so a shard index encoded with one has no size + derivable from metadata alone, and the shard cannot be read. + """ + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """Why this codec cannot be applied to the array that reaches it. @@ -437,6 +453,24 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP return () +def within(prefix: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: + """One entity's problems, located in the document that holds it. + + An entity reports relative to its own `configuration`, so that is what + goes between the field and the member. A problem with an empty + location is about the entity itself -- a malformed `r` name, a + codec that cannot encode what reaches it -- and lands on the field. + """ + return tuple( + ValidationProblem( + (*prefix, *(("configuration", *found.loc) if len(found.loc) != 0 else ())), + found.message, + found.kind, + ) + for found in problems + ) + + def named_configuration( value: object, ) -> tuple[str | None, Mapping[str, object] | None, bool]: @@ -491,4 +525,5 @@ def named_configuration( "one_of", "problem", "sequence_of", + "within", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index e9c9ae79ec..7a51307a8c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -96,8 +96,17 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] Out of scope is not an error: an unknown name may be an extension this reader does not model, and openness means leaving it unjudged. + + The entity has the last word, via `accepts`. Folding is what finds + a candidate -- every `r` spelling is tabled under one invented + identifier -- and the candidate is what says whether the name is + really one of its own. Otherwise the identifier itself would be a + name a document could write. """ - return self.entities.get(field, {}).get(canonical_name(field, name)) + entity = self.entities.get(field, {}).get(canonical_name(field, name)) + if entity is None or not entity.accepts(name): + return None + return entity def coerce( self, field: ExtensionPointField, value: object, loc: Loc = () diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 80ed72ec99..7dd736e6e9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -123,6 +123,7 @@ class BloscCodec(CodecEntity): typesize: int | None = None identifier: ClassVar[str] = BLOSC_CODEC_NAME + variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" # Every member is required but `typesize`, which only means something diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index dd3135754b..4e973b2188 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -21,6 +21,7 @@ is_json_value, one_of, problem, + within, ) from zarr_metadata.v3._parts import ArrayParts @@ -203,17 +204,16 @@ def coerce(cls, value: object, context: "Context") -> Coerced[Self]: codec, problems = super().coerce(value, context) if codec is None: return None, problems - data_type, found = context.coerce(DATA_TYPE, codec.data_type, ("data_type",)) + data_type, found = context.coerce( + DATA_TYPE, codec.data_type, ("configuration", "data_type") + ) return replace(codec, data_type=data_type), (*problems, *found) def problems(self) -> tuple[ValidationProblem, ...]: """Whatever the data type being cast to says about itself.""" if not isinstance(self.data_type, MetadataEntity): return () - return tuple( - ValidationProblem(("data_type", *entry.loc), entry.message, entry.kind) - for entry in self.data_type.problems() - ) + return within(("data_type",), self.data_type.problems()) def configuration(self) -> dict[str, object]: """The target data type in its canonical spelling.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 073c801f65..0bee831002 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -75,6 +75,7 @@ class GzipCodec(CodecEntity): level: int = 5 identifier: ClassVar[str] = GZIP_CODEC_NAME + variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" configuration_required: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 01fe929bfd..0141ce0166 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._entity import ( CODECS, CodecEntity, @@ -22,6 +23,13 @@ problem, sequence_of, ) +from zarr_metadata.v3._parts import ( + UNKNOWN_GRID, + ArrayParts, + ChunkGrid, + shard_index_grid, +) +from zarr_metadata.v3.data_type.uint64 import Uint64DataType if TYPE_CHECKING: from zarr_metadata.v3._registry import Context @@ -138,6 +146,7 @@ class ShardingIndexedCodec(CodecEntity): index_location: ShardingIndexLocation | None = None identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME + variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "array_bytes" configuration_required: ClassVar[bool] = True @@ -153,8 +162,10 @@ def coerce(cls, value: object, context: "Context") -> Coerced[Self]: shard, problems = super().coerce(value, context) if shard is None: return None, problems - inner, from_inner = _coerce_pipeline(shard.codecs, context, ("codecs",)) - index, from_index = _coerce_pipeline(shard.index_codecs, context, ("index_codecs",)) + inner, from_inner = _coerce_pipeline(shard.codecs, context, ("configuration", "codecs")) + index, from_index = _coerce_pipeline( + shard.index_codecs, context, ("configuration", "index_codecs") + ) return ( replace(shard, codecs=inner, index_codecs=index), (*problems, *from_inner, *from_index), @@ -185,6 +196,78 @@ def problems(self) -> tuple[ValidationProblem, ...]: ) return tuple(found) + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + """This shard against the array reaching it, and its two pipelines. + + One sharding configuration encodes every chunk, so its inner + shape has to divide all of them. Under a rectilinear grid an axis + has several lengths and the inner extent must divide each; an axis + whose lengths are unknown declines while the others are judged. + """ + found = list(self._inner_chunk_problems(incoming)) + # Both pipelines start from this codec's own configuration and + # from the spec, so neither waits on what reached the codec. An + # unreadable codec upstream costs the element type and the + # enclosing extents; it does not make the inner chunk shape + # unknown, and the index is a `uint64` array whatever precedes it. + outer = incoming.grid if incoming is not None else UNKNOWN_GRID + found.extend( + chain_problems( + self.codecs, + ArrayParts( + ChunkGrid.regular(self.chunk_shape), + incoming.data_type if incoming is not None else None, + ), + ("codecs",), + ) + ) + found.extend( + chain_problems( + self.index_codecs, + ArrayParts(shard_index_grid(outer, self.chunk_shape), Uint64DataType()), + ("index_codecs",), + ) + ) + found.extend( + ValidationProblem( + ("index_codecs", index), + f"{type(codec).identifier!r} produces variable-size output; " + "index_codecs must be fixed-size", + "invalid_value", + ) + for index, codec in enumerate(self.index_codecs) + if isinstance(codec, CodecEntity) and type(codec).variable_size + ) + return tuple(found) + + def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + """Whether the inner chunk divides every chunk this shard receives.""" + if incoming is None or incoming.grid.rank is None: + return () + if len(self.chunk_shape) != incoming.grid.rank: + return problem( + ("chunk_shape",), + f"chunk_shape has {len(self.chunk_shape)} entries but the incoming array " + f"has {incoming.grid.rank} dimensions", + "invalid_value", + ) + found: list[ValidationProblem] = [] + for position, extent in enumerate(self.chunk_shape): + lengths = incoming.grid.axis(position) + if lengths is None or extent < 1: + continue + indivisible = sorted(length for length in lengths if length % extent != 0) + if len(indivisible) != 0: + found.extend( + problem( + ("chunk_shape", position), + f"inner chunk extent {extent} does not evenly divide the incoming " + f"extent {indivisible[0]}", + "invalid_value", + ) + ) + return tuple(found) + def configuration(self) -> dict[str, object]: """The two pipelines in their canonical spelling, entry by entry.""" members = super().configuration() diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index f59808be0e..65097922ba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -84,6 +84,7 @@ class ZstdCodec(CodecEntity): checksum: bool | None = None identifier: ClassVar[str] = ZSTD_CODEC_NAME + variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" configuration_required: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index a4821d4fd7..20ca6cceba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -18,6 +18,7 @@ MetadataEntity, StorageClass, problem, + within, ) if TYPE_CHECKING: @@ -169,7 +170,7 @@ def coerce(cls, value: object, context: "Context") -> Coerced[Self]: for index, entry in enumerate(cast("tuple[object, ...]", struct.fields)): field = cast("Mapping[str, object]", entry) data_type, from_field = context.coerce( - DATA_TYPE, field["data_type"], ("fields", index, "data_type") + DATA_TYPE, field["data_type"], ("configuration", "fields", index, "data_type") ) found.extend(from_field) fields.append( @@ -235,10 +236,7 @@ def problems(self) -> tuple[ValidationProblem, ...]: "invalid_value", ) ) - found.extend( - ValidationProblem((*at, "data_type", *entry.loc), entry.message, entry.kind) - for entry in field.data_type.problems() - ) + found.extend(within((*at, "data_type"), field.data_type.problems())) return tuple(found) def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/tests/rules/test_canonical.py b/packages/zarr-metadata/tests/rules/test_canonical.py index 04b6b61957..16547752fe 100644 --- a/packages/zarr-metadata/tests/rules/test_canonical.py +++ b/packages/zarr-metadata/tests/rules/test_canonical.py @@ -127,7 +127,9 @@ def test_a_rectilinear_step_is_not_expanded() -> None: def test_error_a_semantically_invalid_document_reports_instead() -> None: result = canonicalize_array_metadata_v3({**BASE, "fill_value": 999}) # type: ignore[arg-type] assert isinstance(result, Invalid) - assert any("fill_value" in problem.message for problem in result.problems) + # The field is the location, not part of the message: the data type + # says what it accepts, and the document says where it was asked. + assert [problem.loc for problem in result.problems] == [("fill_value",)] def test_error_invalid_cannot_be_empty() -> None: From d4ce254b78c3d283a12476290bf58d66a28257c7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:31:18 +0200 Subject: [PATCH 034/107] refactor(zarr-metadata): delete the rule registry the entities replaced 3900 lines. The rule engine existed to dispatch a rule to the entity it was about, gate it on whether that entity's metadata was readable, and propagate what each codec did to the array. Entities now own all three, so there is nothing left to dispatch: - `rules/_registry.py`, `_engine.py`, `_entity.py`: the registry, the `reads`/`reads_optional` gate, `blocking_problems`, `run_rules`. - `rules/_entities/`: nine modules of rules keyed by entity name. - `rules/_v3_array.py`: the fill-value table and the four document rules. - `rules/_storage_class.py`: three frozensets of data type names and a hand-rolled recursion for `struct`. - `rules/_spec.py`, `_chunk_grid.py`, `_pipeline.py`: moved to `v3/_parts.py` and `v3/_chain.py`, minus the name lookups. - `v3/_shape.py`: four tables of per-member type checks, and the six functions that read them. The two remaining cross-field checks that belong to no entity -- v2's `chunks` against `shape`, and a group's inline consolidated children -- are now plain functions. `Rule`, `RuleCheck`, `run_rules`, `applicable` and the six rule-set constants leave the public API with them. Canonicalization collapses to asking each entity for its own `to_json`, plus the one field no entity owns. Three test modules go with the machinery they tested. The property tests over valid documents and corrupted chains stay, and they were what established that the replacement agrees. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/rules/__init__.py | 14 - .../src/zarr_metadata/rules/_canonical.py | 98 +-- .../src/zarr_metadata/rules/_chunk_grid.py | 227 ----- .../src/zarr_metadata/rules/_documents.py | 53 +- .../src/zarr_metadata/rules/_engine.py | 96 --- .../zarr_metadata/rules/_entities/__init__.py | 27 - .../rules/_entities/bytes_codec.py | 51 -- .../rules/_entities/cast_value.py | 43 - .../rules/_entities/numpy_time.py | 43 - .../rules/_entities/rectilinear_grid.py | 131 --- .../rules/_entities/regular_grid.py | 66 -- .../rules/_entities/scale_offset.py | 44 - .../zarr_metadata/rules/_entities/sharding.py | 172 ---- .../rules/_entities/struct_dtype.py | 136 --- .../rules/_entities/transpose.py | 63 -- .../src/zarr_metadata/rules/_entity.py | 55 -- .../src/zarr_metadata/rules/_pipeline.py | 119 --- .../src/zarr_metadata/rules/_registry.py | 358 -------- .../src/zarr_metadata/rules/_spec.py | 148 ---- .../src/zarr_metadata/rules/_storage_class.py | 141 --- .../src/zarr_metadata/rules/_v2_array.py | 47 +- .../src/zarr_metadata/rules/_v3_array.py | 448 ---------- .../src/zarr_metadata/rules/_v3_group.py | 97 ++- .../src/zarr_metadata/v3/_shape.py | 806 ------------------ .../tests/rules/test_chunk_grid.py | 36 +- .../tests/rules/test_registry.py | 271 ------ .../tests/rules/test_spec_propagation.py | 158 ---- .../tests/rules/test_v3_array_rules.py | 4 +- .../zarr-metadata/tests/test_public_api.py | 2 - .../tests/test_registry_drift.py | 105 --- 30 files changed, 148 insertions(+), 3911 deletions(-) delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_engine.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_entity.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_registry.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_spec.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_shape.py delete mode 100644 packages/zarr-metadata/tests/rules/test_registry.py delete mode 100644 packages/zarr-metadata/tests/rules/test_spec_propagation.py delete mode 100644 packages/zarr-metadata/tests/test_registry_drift.py diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py index 52cc0ce221..9f3dd58e3b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py @@ -29,29 +29,15 @@ validate_group_metadata_v2, validate_group_metadata_v3, ) -from zarr_metadata.rules._engine import Rule, RuleCheck, applicable, run_rules -from zarr_metadata.rules._v2_array import ZARR_V2_ARRAY, ZARR_V2_ARRAY_RULES -from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY, ZARR_V3_ARRAY_RULES -from zarr_metadata.rules._v3_group import ZARR_V3_GROUP, ZARR_V3_GROUP_RULES __all__ = [ - "ZARR_V2_ARRAY", - "ZARR_V2_ARRAY_RULES", - "ZARR_V3_ARRAY", - "ZARR_V3_ARRAY_RULES", - "ZARR_V3_GROUP", - "ZARR_V3_GROUP_RULES", "Canonical", "Invalid", - "Rule", - "RuleCheck", - "applicable", "canonicalize_array_metadata_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", "parse_group_metadata_v2", "parse_group_metadata_v3", - "run_rules", "validate_array_metadata_v2", "validate_array_metadata_v3", "validate_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py index 269db46653..eb449b97af 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py @@ -1,24 +1,19 @@ """One document in, one canonical document or one report of why not. -`canonicalize_array_metadata_v3` takes a *syntactically* valid document — +`canonicalize_array_metadata_v3` takes a *syntactically* valid document -- one the model layer has already accepted, so every member is present and -typed as its TypedDict declares — and answers with `Canonical[T] | Invalid`: either the same document in -canonical form or every reason it is not semantically valid. Testing the -literal `valid` field narrows to one or the other. - -Canonical means the simplest spelling with the same meaning, decided per -metadata variety: - -- an entity whose configuration carries nothing collapses to its bare - name, and a `must_understand` of `true` (the default) is dropped while - an explicit `false` is kept, because that one says something. This part - the model layer already performs, so it is delegated rather than - reimplemented. -- `blosc` drops a `typesize` that `shuffle: "noshuffle"` renders ignored. -- a rectilinear dimension's chunk sizes run-length encode, because that - is the spelling that does not grow with the number of chunks. -- `dimension_names` of nothing but nulls says what omitting the field - says. +typed as its TypedDict declares -- and answers with +`Canonical[T] | Invalid`: either the same document in canonical form or +every reason it is not semantically valid. Testing the literal `valid` +field narrows to one or the other. + +Canonical means the simplest spelling with the same meaning, and each +entity decides that for itself in its own `to_json`: an entity whose +configuration carries nothing collapses to its bare name, `blosc` drops a +`typesize` that `shuffle` renders ignored, a rectilinear dimension's chunk +sizes run-length encode. This module only collects the answers, and the +fields no entity owns -- `dimension_names` of nothing but nulls says what +omitting the field says. Two properties are worth holding on to, and `tests/rules/test_canonical.py` asserts both: canonicalizing twice @@ -27,24 +22,20 @@ from __future__ import annotations -from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.rules._documents import validate_array_metadata_v3 -from zarr_metadata.v3._shape import entity_name -from zarr_metadata.v3.chunk_grid.rectilinear import ( - RECTILINEAR_CHUNK_GRID_NAME, - canonical_chunk_shapes, -) -from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME -from zarr_metadata.v3.codec.blosc import canonical_configuration as canonical_blosc +from zarr_metadata.v3._document import read_array_v3 +from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS if TYPE_CHECKING: + from collections.abc import Mapping + from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON - from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearDimSpec DocumentT = TypeVar("DocumentT") @@ -70,43 +61,19 @@ def __post_init__(self) -> None: raise ValueError(msg) -def _canonical_entity(value: object) -> object: - """One entity's configuration in its simplest equivalent form. - - Only varieties with something to say appear here; everything else is - handed to the generic collapse unchanged. - """ - original: object = value - name = entity_name(value) - if name is None or not isinstance(value, Mapping): - return original - entry: Mapping[str, object] = cast("Mapping[str, object]", value) - configuration = entry.get("configuration") - if not isinstance(configuration, Mapping): - return original - members: Mapping[str, object] = cast("Mapping[str, object]", configuration) - if name == BLOSC_CODEC_NAME: - members = canonical_blosc(members) - elif name == RECTILINEAR_CHUNK_GRID_NAME: - shapes = members.get("chunk_shapes") - if isinstance(shapes, tuple): - specs = cast("tuple[RectilinearDimSpec, ...]", shapes) - members = {**members, "chunk_shapes": canonical_chunk_shapes(specs)} - if members is configuration: - return original - return {**entry, "configuration": members} - - def _canonical_document(document: Mapping[str, object]) -> dict[str, object]: - """Per-variety canonicalization, before the generic collapse.""" + """Each entity in its own canonical spelling, and the rest as written.""" + array, _ = read_array_v3(document, CORE_AND_EXTENSIONS) out = dict(document) - for field in ("chunk_grid", "chunk_key_encoding", "data_type"): - if field in out: - out[field] = _canonical_entity(out[field]) - codecs = out.get("codecs") - if isinstance(codecs, tuple): - entries = cast("tuple[object, ...]", codecs) - out["codecs"] = tuple(_canonical_entity(codec) for codec in entries) + for key in ("data_type", "chunk_grid", "chunk_key_encoding"): + entity = getattr(array, key) + if isinstance(entity, MetadataEntity): + out[key] = entity.to_json() + if "codecs" in out: + out["codecs"] = tuple( + codec.to_json() if isinstance(codec, MetadataEntity) else codec + for codec in array.codecs + ) names = out.get("dimension_names") if isinstance(names, tuple) and all( entry is None for entry in cast("tuple[object, ...]", names) @@ -122,16 +89,15 @@ def canonicalize_array_metadata_v3( """`document` in canonical form, or every reason it is not valid. Expects a document the model layer has already accepted. Passing one - it has not is not an error — the composition problems are reported the - same way — but the structural problems come back too, and the result + it has not is not an error -- the semantic problems are reported the + same way -- but the structural problems come back too, and the result is `Invalid` rather than a canonical document. """ problems = validate_array_metadata_v3(document) if len(problems) != 0: return Invalid(problems) canonical = _canonical_document(document) - # The generic collapse — shorthand names, defaulted `must_understand` — - # is what the model layer's round trip already performs. + # The model layer's round trip normalizes the fields no entity owns. return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py deleted file mode 100644 index 1ac924ec6a..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_chunk_grid.py +++ /dev/null @@ -1,227 +0,0 @@ -"""How an array is divided, as far as this package can tell. - -A codec pipeline encodes one chunk, but the same pipeline encodes *every* -chunk, so a rule about a pipeline is a statement about all of them at -once: a shard's inner chunk shape must divide every chunk it will ever be -handed, not some representative one. `ChunkGrid` is what makes that -statement expressible. - -Three pieces of metadata divide an array, and this module is the one place -that reads them: a document's `chunk_grid`, a `sharding_indexed` codec's -`chunk_shape` (a regular grid over the chunk that codec receives), and -that codec's shard index, whose shape the spec derives from the other two. - -Per dimension, and plurally ---------------------------- -`extents` holds one entry per dimension: the set of lengths that -dimension's chunks take. A regular grid gives singletons; a rectilinear -grid gives `{30, 34}` on an axis whose chunks differ; `None` marks an axis -this package cannot read. `rank` survives even when no extent does, -because every chunk of an array has the array's rank whatever divides it. - -Collapsing any of that loses real judgments. A rectilinear grid uniform on -one axis still pins that axis, and a shard is judged there while declining -on the others. - -Prior art ---------- -zarrs builds its grid from metadata *and* the array shape -(`ChunkGrid::create(metadata, array_shape)`) because neither determines a -grid alone, keeps `dimensionality()` total rather than optional, and -reports `chunk_edge_lengths(dimension)` per dimension for the reason -above. Its codec chain distinguishes "no global grid, but one per chunk" -(`ChunkGridMapped::ChunkLocal`) from "nothing known" (`::None`); an -`extents` entry of `None` beside a known `rank` is the per-dimension form -of that distinction. - -- https://github.com/zarrs/zarrs/blob/main/zarrs_chunk_grid/src/lib.rs -- https://github.com/zarrs/zarrs/blob/main/zarrs_codec/src/lib.rs -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, TypeAlias, cast - -from zarr_metadata.rules._entity import entity_configuration -from zarr_metadata.v3._extension_points import CHUNK_GRID -from zarr_metadata.v3._shape import entity_name -from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID_NAME -from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME - -if TYPE_CHECKING: - from collections.abc import Sequence - - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - -Extents: TypeAlias = "tuple[frozenset[int] | None, ...]" -"""One entry per dimension: the lengths that dimension's chunks take. - -A singleton is a uniform axis. `None` is an axis whose lengths this -package cannot determine — distinct from an empty set, which would claim -the axis has no chunks at all. -""" - - -def _positive_int(value: object) -> int | None: - """`value` as a chunk length, or None if it is not a usable one.""" - if isinstance(value, bool) or not isinstance(value, int): - return None - return value if value >= 1 else None - - -def _rank_of(array_shape: object) -> int | None: - """The number of dimensions `array_shape` declares, if it declares any.""" - if not isinstance(array_shape, tuple): - return None - dimensions = cast("tuple[object, ...]", array_shape) - if not all(isinstance(v, int) and not isinstance(v, bool) for v in dimensions): - return None - return len(dimensions) - - -def _uniform(lengths: Sequence[object]) -> Extents: - """Extents for a grid whose chunks are the same everywhere.""" - return tuple( - None if (length := _positive_int(value)) is None else frozenset({length}) - for value in lengths - ) - - -def _rectilinear_axis(spec: object) -> frozenset[int] | None: - """The lengths one rectilinear dimension's chunks take. - - A bare integer is a regular step, so every chunk is that long. An - explicit list names them, with `[size, count]` pairs standing for - repeats; the distinct sizes are what any divisibility question needs. - """ - step = _positive_int(spec) - if step is not None: - return frozenset({step}) - if not isinstance(spec, tuple): - return None - lengths: set[int] = set() - for item in cast("tuple[object, ...]", spec): - size = _positive_int(item) - if size is None and isinstance(item, tuple): - pair = cast("tuple[object, ...]", item) - if len(pair) != 2 or _positive_int(pair[1]) is None: - return None - size = _positive_int(pair[0]) - if size is None: - return None - lengths.add(size) - return frozenset(lengths) if len(lengths) != 0 else None - - -@dataclass(frozen=True, slots=True) -class ChunkGrid: - """The division of an array into the parts a codec pipeline encodes. - - `metadata` is the grid as the document spells it, kept so that a rule - for a grid this package does not model can still read its own - configuration. It is absent for a grid this package derived rather - than read — the regular grid a sharding codec imposes, or a transposed - grid — so nothing may validate it or report a location into it. - """ - - rank: int | None - extents: Extents | None - metadata: ZarrV3MetadataFieldJSON | None = None - - @classmethod - def of(cls, grid: object, array_shape: object) -> ChunkGrid: - """The grid `grid` describes over an array of `array_shape`. - - A grid is not interpretable without the array it divides: the - array shape is what pins the rank when the grid itself cannot be - read, which is the case for every third-party grid. - """ - name = entity_name(grid) - configuration = entity_configuration(CHUNK_GRID, grid) if name is not None else None - metadata = cast("ZarrV3MetadataFieldJSON", grid) if name is not None else None - if configuration is not None: - if name == REGULAR_CHUNK_GRID_NAME: - lengths = configuration.get("chunk_shape") - if isinstance(lengths, tuple): - uniform = _uniform(cast("tuple[object, ...]", lengths)) - return cls(len(uniform), uniform, metadata) - elif name == RECTILINEAR_CHUNK_GRID_NAME: - axes = configuration.get("chunk_shapes") - if isinstance(axes, tuple): - varying = tuple( - _rectilinear_axis(axis) for axis in cast("tuple[object, ...]", axes) - ) - return cls(len(varying), varying, metadata) - rank = _rank_of(array_shape) - return cls(rank, None if rank is None else (None,) * rank, metadata) - - @classmethod - def derived(cls, extents: Extents) -> ChunkGrid: - """A grid this package computed rather than read from a document.""" - return cls(len(extents), extents) - - @classmethod - def regular(cls, lengths: Sequence[object]) -> ChunkGrid: - """The regular grid a sharding codec's `chunk_shape` imposes.""" - return cls.derived(_uniform(lengths)) - - def permuted(self, order: Sequence[int]) -> ChunkGrid: - """This grid with its dimensions reordered by `order`. - - A transposed grid is still a grid — permuting a regular one gives - a regular one — but it is no longer the grid the document wrote, - so the metadata does not survive the trip. - - Declines on anything that is not a permutation of this grid's rank. - The caller checks that too and reports it, but an order is only - shape-validated as a tuple of integers, so this must not be the - thing that decides whether a validator raises `IndexError`. - """ - if self.extents is None or sorted(order) != list(range(len(self.extents))): - return ChunkGrid(self.rank, None) - return ChunkGrid.derived(tuple(self.extents[axis] for axis in order)) - - def axis(self, dimension: int) -> frozenset[int] | None: - """The lengths `dimension`'s chunks take, or None if undetermined.""" - if self.extents is None or dimension >= len(self.extents): - return None - return self.extents[dimension] - - -UNKNOWN_GRID: ChunkGrid = ChunkGrid(None, None) -"""A grid nothing is known about — not even how many dimensions it has.""" - - -def shard_index_grid(shard: ChunkGrid, inner: Sequence[object]) -> ChunkGrid: - """The grid of a shard's index array. - - The spec derives it from the two shapes around it: "The index is an - array with 64-bit unsigned integers with a shape that matches the - chunks per shard tuple with an appended dimension of size 2." The - index is one array rather than a divided one, so each axis holds a - single length — except that under a rectilinear grid the shard itself - varies, so the chunk count varies with it and the axis holds every - value it takes. - """ - inner_extents = _uniform(inner) - trailing: frozenset[int] | None = frozenset({2}) - if shard.extents is None or len(shard.extents) != len(inner_extents): - return ChunkGrid.derived((*(None,) * len(inner_extents), trailing)) - counts: list[frozenset[int] | None] = [] - for lengths, divisor in zip(shard.extents, inner_extents, strict=True): - if lengths is None or divisor is None: - counts.append(None) - continue - step = next(iter(divisor)) - quotients = {length // step for length in lengths if length % step == 0} - counts.append(frozenset(quotients) if len(quotients) == len(lengths) else None) - return ChunkGrid.derived((*counts, trailing)) - - -__all__ = [ - "UNKNOWN_GRID", - "ChunkGrid", - "Extents", - "shard_index_grid", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index c2bee817c4..3c43ce0250 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -28,51 +28,50 @@ from zarr_metadata.model._validation import ( validate_group_metadata_v3 as _validate_group_structure_v3, ) -from zarr_metadata.rules._engine import run_rules -from zarr_metadata.rules._v2_array import ZARR_V2_ARRAY_RULES -from zarr_metadata.rules._v3_group import ZARR_V3_GROUP_RULES +from zarr_metadata.rules._v2_array import array_problems_v2 +from zarr_metadata.rules._v3_group import group_problems_v3 from zarr_metadata.v3._document import array_problems_v3 from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS if TYPE_CHECKING: - from collections.abc import Callable, Sequence + from collections.abc import Callable from zarr_metadata.model._validation import ValidationProblem - from zarr_metadata.rules._engine import Rule from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON _StructuralValidator = Callable[[object], tuple[ValidationProblem, ...]] + _SemanticValidator = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] -def _judged_array_v3(normalized: object) -> tuple[ValidationProblem, ...]: - """Structural and semantic problems in a v3 array document. +def _no_semantics(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """v2 group documents carry no cross-field constraints.""" + return () - The semantic half is `zarr_metadata.v3` asking each entity about - itself and about the parts of the document it meets; this layer - chooses the scope those questions are asked in. + +def _array_semantics_v3(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """The v3 array semantics, in the scope this layer chooses. + + The work is `zarr_metadata.v3` asking each entity about itself and + about the parts of the document it meets; what this layer decides is + which entities are in scope while it asks. """ - problems = _validate_structure_v3(normalized) - if isinstance(normalized, Mapping): - problems = problems + array_problems_v3( - cast("Mapping[str, object]", normalized), CORE_AND_EXTENSIONS - ) - return tuple(problems) + return array_problems_v3(document, CORE_AND_EXTENSIONS) def _judged( - normalized: object, structure: _StructuralValidator, rules: Sequence[Rule] + normalized: object, structure: _StructuralValidator, semantics: _SemanticValidator ) -> tuple[ValidationProblem, ...]: - """Structural and composition problems in an already-normalized document. + """Structural and semantic problems in an already-normalized document. Takes the normalized value rather than the caller's input so that `validate_*` and `parse_*` each walk the document once. """ problems = structure(normalized) if isinstance(normalized, Mapping): - problems = problems + run_rules(rules, cast("Mapping[str, object]", normalized)) + problems = problems + semantics(cast("Mapping[str, object]", normalized)) return tuple(problems) @@ -85,7 +84,7 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: (e.g. fresh `json.loads` output) are judged at the canonical data level rather than rejected for their spelling. """ - return _judged_array_v3(arrays_to_tuples(value)) + return _judged(arrays_to_tuples(value), _validate_structure_v3, _array_semantics_v3) def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: @@ -96,7 +95,7 @@ def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: problem found. """ normalized = arrays_to_tuples(value) - problems = _judged_array_v3(normalized) + problems = _judged(normalized, _validate_structure_v3, _array_semantics_v3) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3ArrayMetadataJSON", normalized) @@ -108,7 +107,7 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: JSON arrays are normalized to tuples before judgment, as in `validate_array_metadata_v3`. """ - return _judged(arrays_to_tuples(value), _validate_structure_v2, ZARR_V2_ARRAY_RULES) + return _judged(arrays_to_tuples(value), _validate_structure_v2, array_problems_v2) def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: @@ -119,7 +118,7 @@ def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: problem found. """ normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_structure_v2, ZARR_V2_ARRAY_RULES) + problems = _judged(normalized, _validate_structure_v2, array_problems_v2) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV2ArrayMetadataJSON", normalized) @@ -132,13 +131,13 @@ def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: consolidated child document invalid under its own rules is reported here, at its path. """ - return _judged(arrays_to_tuples(value), _validate_group_structure_v3, ZARR_V3_GROUP_RULES) + return _judged(arrays_to_tuples(value), _validate_group_structure_v3, group_problems_v3) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: """Return `value` as a valid `ZarrV3GroupMetadataJSON`, or raise.""" normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_group_structure_v3, ZARR_V3_GROUP_RULES) + problems = _judged(normalized, _validate_group_structure_v3, group_problems_v3) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3GroupMetadataJSON", normalized) @@ -150,13 +149,13 @@ def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: v2 group documents carry no composition constraints today, so this is the structural judgment, offered here for a uniform read-side API. """ - return _judged(arrays_to_tuples(value), _validate_group_structure_v2, ()) + return _judged(arrays_to_tuples(value), _validate_group_structure_v2, _no_semantics) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: """Return `value` as a valid `ZarrV2GroupMetadataJSON`, or raise.""" normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_group_structure_v2, ()) + problems = _judged(normalized, _validate_group_structure_v2, _no_semantics) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV2GroupMetadataJSON", normalized) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py b/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py deleted file mode 100644 index 924a705fe2..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_engine.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Rules gated by the document fields they read. - -A `Rule` runs when every key in `requires` is present, so one rule set -serves complete documents and partial ones without imposing field order. -Gating rather than ordering is conventional — Ecto changesets, Clojure -spec, Valibot's `partialCheck`, JSON Schema's `dependentSchemas` — and -has two consequences here. - -**Order-free by construction.** No topological sort, so mutually -dependent rules are expressible. - -**Absence is deliberately inexpressible.** A rule cannot ask whether a -field is missing: that is negation-as-failure, sound only under a -closed-world assumption, and a partially built document is an open world -where the key may still arrive. Required-key checks therefore stay in -structural validation. - -Rules may receive structurally invalid values. A rule that cannot safely -interpret its inputs leaves the problem to structural validation. -""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator, Mapping, Sequence -from collections.abc import Set as AbstractSet -from dataclasses import dataclass -from typing import cast - -from zarr_metadata.model._validation import ValidationProblem - -RuleCheck = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] -"""A rule's check: the whole document in, every problem it finds out.""" - - -@dataclass(frozen=True, slots=True) -class Rule: - """One composition check over a (possibly partial) metadata document. - - `requires` are the document keys the check reads; the rule fires only - when all of them are present. `check` receives the whole document (so - coupled fields are examined together) and returns every problem it - finds, empty when the rule passes. - """ - - requires: frozenset[str] - check: RuleCheck - - -def applicable(rules: Sequence[Rule], present: AbstractSet[str]) -> Iterator[Rule]: - """The subset of `rules` whose required keys are all present.""" - return (rule for rule in rules if rule.requires <= present) - - -def run_rules( - rules: Sequence[Rule], document: Mapping[str, object] -) -> tuple[ValidationProblem, ...]: - """Run every applicable rule over `document`, collecting all problems.""" - problems: list[ValidationProblem] = [] - for rule in applicable(rules, document.keys()): - problems.extend(rule.check(document)) - return tuple(problems) - - -def as_string_mapping(value: object) -> Mapping[str, object] | None: - """`value` as a string-keyed mapping, or None if it is not one.""" - if not isinstance(value, Mapping): - return None - mapping = cast("Mapping[object, object]", value) - if any(not isinstance(key, str) for key in mapping): - return None - return cast("Mapping[str, object]", mapping) - - -def as_sequence(value: object) -> Sequence[object] | None: - """`value` as a JSON-array-shaped sequence, or None if it is not one.""" - if isinstance(value, (list, tuple)): - return cast("Sequence[object]", value) - return None - - -def prefixed( - loc: tuple[str | int, ...], problems: Sequence[ValidationProblem] -) -> tuple[ValidationProblem, ...]: - """Re-base every problem's `loc` under `loc` (for nested documents).""" - return tuple( - ValidationProblem((*loc, *problem.loc), problem.message, problem.kind) - for problem in problems - ) - - -__all__ = [ - "Rule", - "RuleCheck", - "applicable", - "run_rules", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py deleted file mode 100644 index a1662ccc1f..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Per-entity composition rules, discovered automatically. - -Every module here owns the rules for one codec or chunk grid and -registers them with `zarr_metadata.rules._registry` at import time. -Adding a new entity means adding a module here and nothing else: this -package imports every sibling module on import, so there is no -registration list to update and no document-rule module to edit. - -That auto-discovery is the deliberate answer to two failure modes. A -hand-written registry lets a rule be defined and never registered, and a -hand-written import list lets a whole module be defined and never -imported; both produce rules that silently never run. `tests/rules/ -test_registry.py` closes the remaining gap by asserting that every codec -and chunk grid the package models is either registered here or listed as -deliberately rule-free. -""" - -from __future__ import annotations - -import importlib -import pkgutil - -for _module in pkgutil.iter_modules(__path__): - if not _module.name.startswith("_"): - importlib.import_module(f"{__name__}.{_module.name}") - -__all__: list[str] = [] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py deleted file mode 100644 index 87dc3601ee..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Composition rules for the core ``bytes`` codec.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.rules._storage_class import data_type_name, storage_class -from zarr_metadata.v3._extension_points import CODECS, DATA_TYPE -from zarr_metadata.v3._shape import blocking_problems, validate_known_entity_metadata -from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" - - -@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME, reads_optional=frozenset({"endian"})) -def data_type_has_a_raw_byte_representation( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - if incoming is None or incoming.data_type is None: - return () - shape_verdict = validate_known_entity_metadata(DATA_TYPE, incoming.data_type) - if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0: - return () - found = storage_class(incoming.data_type) - name = data_type_name(incoming.data_type) - if found == "variable_length": - return ( - ValidationProblem( - (), - f"bytes codec is not compatible with variable-length data_type {name!r}", - "invalid_value", - ), - ) - if found == "multi_byte" and "endian" not in configuration: - # Name the type: inside a shard's `index_codecs` the array is the - # shard index, whose uint64 type appears nowhere in the document. - return ( - ValidationProblem( - ("endian",), - f"endian is required for data type {name!r}, which contains multi-byte values", - "missing_key", - ), - ) - return () diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py deleted file mode 100644 index 8e328392ca..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Spec transition for the `cast_value` codec. - -`cast_value` validates its target data type and -changes the data type everything downstream receives: a later rule that -reads the type (the `bytes` codec's endianness requirement, for example) -must judge against the configured target. - -The codec also casts the fill value, and the spec makes a failed -round-trip a MUST error. Deciding that means implementing the cast -(rounding modes, out-of-range clamp and wrap, scalar maps), which is -numeric semantics rather than JSON judgment; it belongs to whatever -implements the codec and is not modelled here. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.rules._registry import entity_rule, run_entity_rules -from zarr_metadata.rules._spec import ArrayParts, spec_transition -from zarr_metadata.v3._extension_points import CODECS, DATA_TYPE -from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.model._validation import ValidationProblem - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - - -@entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME, reads=frozenset({"data_type"})) -def target_data_type_obeys_its_rules( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """A cast target obeys the same entity rules as a top-level data type.""" - return run_entity_rules(DATA_TYPE, configuration["data_type"], document, ("data_type",)) - - -@spec_transition(CAST_VALUE_CODEC_NAME) -def cast_data_type(configuration: Mapping[str, object], incoming: ArrayParts) -> ArrayParts: - """The outgoing type is the configured target.""" - target = cast("ZarrV3MetadataFieldJSON", configuration["data_type"]) - return incoming.with_data_type(target) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py deleted file mode 100644 index c18a179315..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Composition rules shared by NumPy datetime and timedelta data types.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.v3._extension_points import DATA_TYPE -from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" -_MAX_SCALE_FACTOR = 2**31 - 1 -_SCALE_FACTOR = frozenset({"scale_factor"}) - - -def _scale_factor_is_in_range( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - scale_factor = cast("int", configuration["scale_factor"]) - if 1 <= scale_factor <= _MAX_SCALE_FACTOR: - return () - return ( - ValidationProblem( - ("scale_factor",), - f"expected an integer in [1, {_MAX_SCALE_FACTOR}], got {scale_factor}", - "invalid_value", - ), - ) - - -entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE_NAME, reads=_SCALE_FACTOR)( - _scale_factor_is_in_range -) -entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE_NAME, reads=_SCALE_FACTOR)( - _scale_factor_is_in_range -) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py deleted file mode 100644 index c6a70eb58a..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Composition rules for the `rectilinear` chunk grid.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.v3._extension_points import CHUNK_GRID -from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts, Sequence - -_ARRAY_V3 = "zarr_v3_array" - - -def _is_int(value: object) -> bool: - return isinstance(value, int) and not isinstance(value, bool) - - -def _expanded_extent(spec: Sequence[object]) -> int | None: - """The total extent an explicit dimension spec covers, or None. - - Entries are chunk sizes or `[size, count]` run-length pairs. Answers - None when any entry is non-positive — the values rule owns that - complaint, and a sum over bad entries would be noise. - """ - total = 0 - for item in spec: - if _is_int(item) and cast(int, item) >= 1: - total += cast(int, item) - elif isinstance(item, tuple): - size, count = cast("tuple[object, object]", item) - if not (_is_int(size) and _is_int(count)): - return None - if cast(int, size) < 1 or cast(int, count) < 1: - return None - total += cast(int, size) * cast(int, count) - else: - return None - return total - - -@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shapes"})) -def chunk_extents_are_positive( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """Every chunk extent, bare or run-length encoded, must be positive.""" - chunk_shapes = cast("tuple[object, ...]", configuration["chunk_shapes"]) - problems: list[ValidationProblem] = [] - for dim, spec in enumerate(chunk_shapes): - loc: tuple[str | int, ...] = ("chunk_shapes", dim) - if _is_int(spec): - if cast(int, spec) < 1: - problems.append( - ValidationProblem( - loc, f"expected a positive chunk extent, got {spec}", "invalid_value" - ) - ) - continue - if not isinstance(spec, tuple): - continue - for position, item in enumerate(cast("tuple[object, ...]", spec)): - if _is_int(item) and cast(int, item) < 1: - problems.append( - ValidationProblem( - (*loc, position), - f"expected a positive chunk extent, got {item}", - "invalid_value", - ) - ) - elif isinstance(item, tuple): - size, count = cast("tuple[int, int]", item) - if size < 1 or count < 1: - problems.append( - ValidationProblem( - (*loc, position), - f"expected a positive [size, count] pair, got {item!r}", - "invalid_value", - ) - ) - return tuple(problems) - - -@entity_rule( - _ARRAY_V3, - CHUNK_GRID, - RECTILINEAR_CHUNK_GRID_NAME, - requires=frozenset({"shape"}), - reads=frozenset({"chunk_shapes"}), -) -def tiles_the_array( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """One spec per dimension, and explicit specs must sum to that extent. - - A bare-integer dimension spec is uniform shorthand and imposes no sum - constraint; an explicit list of chunk sizes must tile its dimension - exactly. - """ - shape = document["shape"] - if not isinstance(shape, (list, tuple)): - return () - extents = cast("tuple[object, ...]", shape) - chunk_shapes = cast("tuple[object, ...]", configuration["chunk_shapes"]) - if len(chunk_shapes) != len(extents): - return ( - ValidationProblem( - ("chunk_shapes",), - f"chunk_shapes has {len(chunk_shapes)} entries but shape has " - f"{len(extents)} dimensions", - "invalid_value", - ), - ) - problems: list[ValidationProblem] = [] - for dim, (spec, extent) in enumerate(zip(chunk_shapes, extents, strict=True)): - if not _is_int(extent) or _is_int(spec) or not isinstance(spec, tuple): - continue - total = _expanded_extent(cast("tuple[object, ...]", spec)) - if total is not None and total < cast("int", extent): - problems.append( - ValidationProblem( - ("chunk_shapes", dim), - f"chunk sizes sum to {total} but must cover shape[{dim}] extent {extent}", - "invalid_value", - ) - ) - return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py deleted file mode 100644 index e1140ddd9f..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Composition rules for the `regular` chunk grid.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.v3._extension_points import CHUNK_GRID -from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" - - -@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shape"})) -def chunk_extents_are_positive( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """Every chunk extent must be at least one element. - - A zero extent makes the chunk index `floor(i / 0)` undefined; a - negative one is meaningless. The shape validator enforces that the - entries are integers, so this rule judges only their values. - """ - chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(chunk_shape) - if extent < 1 - ) - - -@entity_rule( - _ARRAY_V3, - CHUNK_GRID, - REGULAR_CHUNK_GRID_NAME, - requires=frozenset({"shape"}), - reads=frozenset({"chunk_shape"}), -) -def chunks_every_dimension( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """A regular grid must chunk every array dimension.""" - shape = document["shape"] - if not isinstance(shape, (list, tuple)): - return () - chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) - if len(chunk_shape) == len(cast("tuple[object, ...]", shape)): - return () - return ( - ValidationProblem( - ("chunk_shape",), - f"chunk_shape has {len(chunk_shape)} entries but shape has " - f"{len(cast('tuple[object, ...]', shape))} dimensions", - "invalid_value", - ), - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py deleted file mode 100644 index 734f418a3e..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/scale_offset.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Spec transition for the `scale_offset` codec. - -`scale_offset` subtracts an offset and multiplies by a scale, element by -element, and the spec requires the result to be representable in the -array's own data type: "The encoding and decoding transformations MUST be -performed using the arithmetic semantics of the input array's data type. -If any intermediate or final value produced during encoding or decoding -is not representable in that data type, implementations MUST treat this -as an error." - -So it changes neither the shape nor the data type, and its transition is -the identity. The spec is explicit that narrowing is somebody else's job: -the codec's `astype` field "was removed from the `scale_offset` codec in -favor of expressing data type conversion via a dedicated codec", because -"in Zarr V3, a `dtype` field is not needed — the data type of the input -to an array-array codec is determined by its location in the `codecs` -metadata". - -Registering this matters beyond tidiness. A modelled `array -> array` -codec with no transition is treated as unknown, which stops propagation -and silently stands down every rule downstream of it — so without this, -inserting a no-op `scale_offset` would switch off the `bytes` codec's -endianness requirement and every shard's geometry check. - -https://github.com/zarr-developers/zarr-extensions/blob/main/codecs/scale_offset/README.md -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from zarr_metadata.rules._spec import spec_transition -from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.rules._spec import ArrayParts - - -@spec_transition(SCALE_OFFSET_CODEC_NAME) -def preserves_the_array(configuration: Mapping[str, object], incoming: ArrayParts) -> ArrayParts: - """Element-wise arithmetic in the input type: same shape, same type.""" - return incoming diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py deleted file mode 100644 index 86b8fc64d5..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Composition rules for the `sharding_indexed` codec. - -Sharding is the one entity whose configuration contains whole pipelines -and its own geometry, so its rules recurse: the inner `codecs` and -`index_codecs` are judged by the same pipeline checks that judge the -document's top-level `codecs`, at every nesting depth. - -Every geometry judgment here is against the *incoming* array spec — the -array as transformed by every codec before this one — never against the -document's chunk grid directly. A `transpose` in front of a shard changes -which extents the shard has to divide, and reading the grid instead gave -wrong verdicts in both directions: it accepted an inner chunk that did -not divide the transposed shape and rejected one that did. - -The inner pipeline receives the inner chunk as its incoming spec (with -the incoming data type carried through), so a transpose or nested shard -inside it is judged against the inner chunk, recursively — each sharding -level encloses the next. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._chunk_grid import UNKNOWN_GRID, ChunkGrid, shard_index_grid -from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems -from zarr_metadata.rules._registry import entity_rule, run_chain_rules -from zarr_metadata.rules._spec import ArrayParts -from zarr_metadata.v3._extension_points import CODECS -from zarr_metadata.v3._shape import entity_name -from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME -from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME -from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC_NAME -from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - -_ARRAY_V3 = "zarr_v3_array" -_CHUNK_SHAPE = frozenset({"chunk_shape"}) -_PIPELINES = frozenset({"chunk_shape", "codecs", "index_codecs"}) -_INDEX_CODECS = frozenset({"index_codecs"}) -_VARIABLE_SIZE_CODECS = frozenset( - {BLOSC_CODEC_NAME, GZIP_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME, ZSTD_CODEC_NAME} -) - - -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE) -def inner_chunk_extents_are_positive( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - chunk_shape = cast("tuple[int, ...]", configuration["chunk_shape"]) - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(chunk_shape) - if extent < 1 - ) - - -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE) -def inner_chunks_tile_the_incoming_array( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """The inner chunk must rank-match and divide every chunk it receives. - - One sharding configuration encodes every chunk, so its inner shape has - to divide all of them. Under a rectilinear grid an axis has several - lengths and the inner extent must divide each; an axis whose lengths - are unknown declines while the others are still judged. - """ - if incoming is None or incoming.grid.rank is None: - return () - inner = cast("tuple[int, ...]", configuration["chunk_shape"]) - if len(inner) != incoming.grid.rank: - return ( - ValidationProblem( - ("chunk_shape",), - f"chunk_shape has {len(inner)} entries but the incoming array has " - f"{incoming.grid.rank} dimensions", - "invalid_value", - ), - ) - problems: list[ValidationProblem] = [] - for position, extent in enumerate(inner): - lengths = incoming.grid.axis(position) - if lengths is None or extent < 1: - continue - indivisible = sorted(length for length in lengths if length % extent != 0) - if len(indivisible) != 0: - problems.append( - ValidationProblem( - ("chunk_shape", position), - f"inner chunk extent {extent} does not evenly divide the incoming " - f"extent {indivisible[0]}", - "invalid_value", - ) - ) - return tuple(problems) - - -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_PIPELINES) -def inner_pipelines_are_pipelines( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """`codecs` and `index_codecs` obey the pipeline rules, recursively. - - Both get the ordering and shape judgments the top-level pipeline gets, - plus the entity rules of whatever codecs appear inside. The inner - `codecs` chain starts from the inner chunk with the incoming data - type; a nested shard or transpose inside it is therefore judged - against the inner chunk, and its own transitions carry on from there. - The `index_codecs` chain encodes the shard index: a `uint64` array of - chunks-per-shard plus a trailing dimension of 2, derived by - `zarr_metadata.rules._chunk_grid.shard_index_grid`. - """ - inner = configuration["chunk_shape"] - if not isinstance(inner, tuple): - inner_start: ArrayParts | None = None - index_start: ArrayParts | None = None - else: - extents = cast("tuple[object, ...]", inner) - # A shard is a nested array: `chunk_shape` is a regular grid over - # the chunk this codec receives, so the inner pipeline is built - # exactly like the document's own, and the index's grid follows - # from the two together. - # - # Both come from this codec's own configuration and from the spec, - # so neither waits on what reached the codec. An unreadable codec - # upstream costs the element type and the enclosing extents; it - # does not make the inner chunk shape unknown, and the index is a - # `uint64` array whatever happens before it. - outer = incoming.grid if incoming is not None else UNKNOWN_GRID - inner_start = ArrayParts( - ChunkGrid.regular(extents), incoming.data_type if incoming is not None else None - ) - index_start = ArrayParts(shard_index_grid(outer, extents), "uint64") - problems: list[ValidationProblem] = [] - for key in ("codecs", "index_codecs"): - entries = configuration[key] - if not isinstance(entries, (list, tuple)): - continue - sequence = cast("tuple[object, ...]", entries) - problems.extend(pipeline_order_problems(sequence, (key,))) - problems.extend(shape_problems(sequence, (key,))) - # The index pipeline encodes the shard index, not the array: a - # uint64 array of offsets and lengths, so e.g. the bytes codec - # inside it still needs an endianness. - start = inner_start if key == "codecs" else index_start - problems.extend(run_chain_rules(CODECS, sequence, document, (key,), start)) - return tuple(problems) - - -@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_INDEX_CODECS) -def index_codecs_have_fixed_encoded_size( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """The shard index must have an encoded size derivable from metadata.""" - entries = cast("tuple[object, ...]", configuration["index_codecs"]) - return tuple( - ValidationProblem( - ("index_codecs", index), - f"{name!r} produces variable-size output; index_codecs must be fixed-size", - "invalid_value", - ) - for index, entry in enumerate(entries) - if (name := entity_name(entry)) in _VARIABLE_SIZE_CODECS - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py deleted file mode 100644 index f9e0650aab..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Composition rules for the `struct` data type. - -`StructField`'s own docstring promises field names are unique within a -struct and non-empty. Neither is expressible in a TypedDict, so both are -composition judgments and live here. - -The fixed-size field rule is the `bytes` codec's question asked from the -other side — the spec writes it as "Variable-length data types (e.g. -`"string"`) MUST NOT be used as field types, as they do not have a fixed -encoded size" — so it defers to the shared classifier in -`zarr_metadata.rules._storage_class` rather than keeping a second table -of data-type sizes. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._engine import as_string_mapping -from zarr_metadata.rules._registry import entity_rule, run_entity_rules -from zarr_metadata.rules._storage_class import storage_class -from zarr_metadata.v3._extension_points import DATA_TYPE -from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME - -if TYPE_CHECKING: - from zarr_metadata.rules._spec import ArrayParts - -_ARRAY_V3 = "zarr_v3_array" - - -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) -def field_data_types_obey_their_rules( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """Apply every known data type's rules inside struct fields, recursively.""" - problems: list[ValidationProblem] = [] - for index, field in enumerate(cast("tuple[object, ...]", configuration["fields"])): - field_mapping = as_string_mapping(field) - if field_mapping is not None: - problems.extend( - run_entity_rules( - DATA_TYPE, - field_mapping.get("data_type"), - document, - ("fields", index, "data_type"), - ) - ) - return tuple(problems) - - -def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str], ...]: - """`(index, name)` for each field with a string name, else nothing. - - Anything the shape validator would reject is skipped: it owns that - complaint, and judging names inside a malformed field list is noise. - """ - fields = configuration.get("fields") - if not isinstance(fields, tuple): - return () - named: list[tuple[int, str]] = [] - for index, field in enumerate(cast("tuple[object, ...]", fields)): - if not isinstance(field, Mapping): - continue - name = cast("Mapping[object, object]", field).get("name") - if isinstance(name, str): - named.append((index, name)) - return tuple(named) - - -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) -def fields_are_non_empty( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - fields = cast("tuple[object, ...]", configuration["fields"]) - if len(fields) != 0: - return () - return (ValidationProblem(("fields",), "expected at least one struct field", "invalid_value"),) - - -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) -def field_data_types_are_fixed_size( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - fields = cast("tuple[object, ...]", configuration["fields"]) - problems: list[ValidationProblem] = [] - for index, field in enumerate(fields): - field_mapping = as_string_mapping(field) - if field_mapping is None or "data_type" not in field_mapping: - continue - if storage_class(field_mapping["data_type"]) == "variable_length": - problems.append( - ValidationProblem( - ("fields", index, "data_type"), - "struct fields must use fixed-size data types", - "invalid_value", - ) - ) - return tuple(problems) - - -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) -def field_names_are_non_empty( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """A struct field must be addressable, so its name cannot be empty.""" - return tuple( - ValidationProblem( - ("fields", index, "name"), "expected a non-empty field name", "invalid_value" - ) - for index, name in _field_names(configuration) - if name == "" - ) - - -@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"})) -def field_names_are_unique( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """Duplicate field names make a fill value's per-field mapping ambiguous.""" - seen: dict[str, int] = {} - problems: list[ValidationProblem] = [] - for index, name in _field_names(configuration): - first = seen.get(name) - if first is None: - seen[name] = index - continue - problems.append( - ValidationProblem( - ("fields", index, "name"), - f"duplicate field name {name!r}, already used by field {first}", - "invalid_value", - ) - ) - return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py deleted file mode 100644 index 040c9403d3..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entities/transpose.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Composition rules and spec transition for the `transpose` codec. - -Whether `order` is a permutation of its own indices is a fact about the -value, checked by `v3._shape`. What is left here needs the array that -reached the codec. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._chunk_grid import ChunkGrid -from zarr_metadata.rules._registry import entity_rule -from zarr_metadata.rules._spec import ArrayParts, spec_transition -from zarr_metadata.v3._extension_points import CODECS -from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME - -if TYPE_CHECKING: - from collections.abc import Mapping - -_ARRAY_V3 = "zarr_v3_array" - - -@spec_transition(TRANSPOSE_CODEC_NAME) -def permute_grid(configuration: Mapping[str, object], incoming: ArrayParts) -> ArrayParts: - """The outgoing grid is the incoming one with its axes reordered. - - A transposed grid is still a grid, so the parts survive the codec with - their lengths permuted. An order that is not a permutation of the rank - yields a grid of unknown extents — the rules below report the order - itself, and extents derived from a bad order would be a guess. - """ - order = cast("tuple[int, ...]", configuration["order"]) - if sorted(order) != list(range(len(order))): - return incoming.with_grid(ChunkGrid(incoming.grid.rank, None)) - return incoming.with_grid(incoming.grid.permuted(order)) - - -@entity_rule(_ARRAY_V3, CODECS, TRANSPOSE_CODEC_NAME, reads=frozenset({"order"})) -def order_matches_incoming_rank( - configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArrayParts | None -) -> tuple[ValidationProblem, ...]: - """A transpose permutes the array it receives, so ranks must agree. - - Judged against what actually reaches this codec, not the document's - `shape`: inside a shard that is the inner chunk, and after another - transpose it is that transpose's output. Declines when the rank is - unknown. - """ - rank = incoming.grid.rank if incoming is not None else None - if rank is None: - return () - order = cast("tuple[int, ...]", configuration["order"]) - if len(order) == rank: - return () - return ( - ValidationProblem( - ("order",), - f"order has {len(order)} entries but the incoming array has {rank} dimensions", - "invalid_value", - ), - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_entity.py b/packages/zarr-metadata/src/zarr_metadata/rules/_entity.py deleted file mode 100644 index 955a186e3e..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_entity.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Reading a named entity's configuration. - -An entity's `configuration` is only worth reading once its shape has been -vouched for, and there are two useful strictnesses. `entity_configuration` -is all-or-nothing, for callers that derive something from the whole -configuration (a spec transition). `run_entity_rules` wants the finer -per-member judgment and reaches for `configuration_mapping` plus the shape -verdict directly. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from zarr_metadata.rules._engine import as_string_mapping -from zarr_metadata.v3._shape import blocking_problems, validate_known_entity_metadata - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.v3._extension_points import ExtensionPointField - - -def entity_configuration(field: ExtensionPointField, value: object) -> Mapping[str, object] | None: - """`value`'s configuration if every modelled field is usable, else None. - - The all-or-nothing gate `propagate` needs: a spec transition reads the - configuration to compute what the next codec receives, so one unusable - member makes the whole outgoing spec a guess. `run_entity_rules` uses - the finer per-member gate instead. `unknown_key` problems do not make - an entity unusable; anything else does. - """ - verdict = validate_known_entity_metadata(field, value) - if verdict is None or len(blocking_problems(verdict)) != 0: - return None - return configuration_mapping(value) - - -def configuration_mapping(value: object) -> Mapping[str, object] | None: - """`value`'s configuration mapping, with no judgment of its contents.""" - mapping = as_string_mapping(value) - if mapping is None: - # Bare-string metadata is the canonical spelling for entities whose - # configuration is optional. Rules still need a real mapping to run - # against, especially when they judge a missing optional member. - return {} if isinstance(value, str) else None - if "configuration" not in mapping: - return {} - return as_string_mapping(mapping["configuration"]) - - -__all__ = [ - "configuration_mapping", - "entity_configuration", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py deleted file mode 100644 index 34a05deb50..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_pipeline.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Codec-pipeline judgments, shared by the array rules and by sharding. - -A sharding codec's `codecs` and `index_codecs` are pipelines exactly like -the document's top-level `codecs`, so the ordering and shape checks live -here rather than in either caller: sharding recurses into them at every -nesting depth, and the top-level array rules apply them at depth zero. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Final - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._engine import as_string_mapping, prefixed -from zarr_metadata.v3._shape import entity_name, validate_known_codec_metadata -from zarr_metadata.v3.codec.kind import codec_kind_of_name - -if TYPE_CHECKING: - from collections.abc import Sequence - - from zarr_metadata.v3.codec.kind import CodecKind - -_KIND_RANK: Final = {"array_array": 0, "array_bytes": 1, "bytes_bytes": 2} - - -def codec_kind(codec: object) -> CodecKind | None: - """The pipeline kind of `codec`, classified by name alone. - - Spelling-insensitive on purpose: a known codec in an invalid spelling - still ranks as its kind, so two spellings of the same pipeline always - get the same ordering verdict and a misspelled known codec is never - mistaken for an unknown extension (which would suppress the - exactly-one-`array->bytes` count). - """ - name = entity_name(codec) - if name is None: - return None - return codec_kind_of_name(name) - - -def _codec_label(codec: object) -> str: - if isinstance(codec, str): - return repr(codec) - mapping = as_string_mapping(codec) - if mapping is not None: - return repr(mapping.get("name")) - return repr(codec) - - -def pipeline_order_problems( - entries: Sequence[object], loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - """The spec pipeline shape: `array->array`* `array->bytes` `bytes->bytes`*. - - Codecs of genuinely unknown name are skipped: they impose no ordering - constraint, and their presence makes the exactly-one-`array->bytes` - count inconclusive (an unknown codec might be the pipeline's - `array->bytes` stage), so that check only fires when every codec is - classified. - """ - - problems: list[ValidationProblem] = [] - kinds = [codec_kind(codec) for codec in entries] - max_rank_seen = -1 - array_bytes_count = 0 - for index, (codec, kind) in enumerate(zip(entries, kinds, strict=True)): - if kind is None: - continue - rank = _KIND_RANK[kind] - if rank < max_rank_seen: - problems.append( - ValidationProblem( - (*loc, index), - f"{kind.replace('_', '->')} codec {_codec_label(codec)} may not " - "follow a later-stage codec in the pipeline", - "invalid_value", - ) - ) - max_rank_seen = max(max_rank_seen, rank) - if kind == "array_bytes": - array_bytes_count += 1 - if array_bytes_count > 1: - problems.append( - ValidationProblem( - (*loc, index), - f"extra array->bytes codec {_codec_label(codec)}: a pipeline " - "has exactly one", - "invalid_value", - ) - ) - if array_bytes_count == 0 and all(kind is not None for kind in kinds): - problems.append( - ValidationProblem(loc, "codec pipeline has no array->bytes codec", "invalid_value") - ) - return tuple(problems) - - -def shape_problems( - entries: Sequence[object], loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - """Shape problems for every known-name codec entry in `entries`. - - Unknown names pass untouched (extension openness); entries without an - interpretable name decline in favor of the structural validator. - """ - problems: list[ValidationProblem] = [] - for index, codec in enumerate(entries): - found = validate_known_codec_metadata(codec) - # None is "not a known codec" (unjudged); () is "known and valid". - if found is not None: - problems.extend(prefixed((*loc, index), found)) - return tuple(problems) - - -__all__ = [ - "codec_kind", - "pipeline_order_problems", - "shape_problems", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py b/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py deleted file mode 100644 index 88c9b76fcb..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_registry.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Register rules by document type and extension entity. - -`@document_rule` and `@entity_rule` register checks where they are -defined, so a rule cannot be written without joining the set it belongs -to. Both reject dependencies absent from the document type. Entity rules -are keyed by `(field, canonical_name)` and require a corresponding shape -validator. -""" - -from __future__ import annotations - -from collections import defaultdict -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass -from typing import TYPE_CHECKING, Final, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.rules._chunk_grid import ChunkGrid -from zarr_metadata.rules._engine import Rule -from zarr_metadata.rules._entity import configuration_mapping, entity_configuration -from zarr_metadata.rules._spec import ArrayParts, propagate -from zarr_metadata.v3._extension_points import ExtensionPointField, canonical_name - -if TYPE_CHECKING: - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - -from zarr_metadata.v3._shape import ( - blocking_problems, - entity_configuration_keys, - entity_name, - entity_required_configuration_keys, - modelled_entities, - validate_known_entity_metadata, -) - -EntityCheck = Callable[ - [Mapping[str, object], Mapping[str, object], "ArrayParts | None"], - "tuple[ValidationProblem, ...]", -] -"""An entity rule's check: `(configuration, document, incoming)` in, problems out. - -`incoming` is what the entity receives — for a codec, the array parts as -transformed by every codec before it in the chain, or `None` where this -package can no longer say. A caller with no chain context passes `None`. -Rules that need it test for `None` and decline; rules that do not simply -ignore it. - -Problems carry locations relative to the entity's `configuration`; the -dispatcher re-bases them onto the entity's position in the document. -""" - - -@dataclass(frozen=True, slots=True) -class EntityRule: - """One composition check for a named extension entity. - - Identified by `(field, entity)`, never by name alone: names are - unique only within an extension point, and `bytes` is both a core - codec and a registered extension data type. Keying by name would - make a rule written for one fire on the other. - - `requires` are *document* keys the check reads beyond the entity - itself (e.g. `shape`), gating the rule exactly as `Rule.requires` - does. - - `reads` are the *required* configuration members the check subscripts. - A rule runs only when none of them has a shape problem of its own, - which is what makes `configuration["level"]` safe: a required member - that is absent or ill-typed is reported at `("configuration", member)` - and stands the rule down, while the rest of the entity is still - judged. Only required members may be declared here — an optional one - can be absent with nothing reported, so subscripting it would raise - out of a validator. - - `reads_optional` are modelled members the check tests for presence - rather than subscripting (`"endian" not in configuration`). They gate - the rule the same way; they are separate so that the subscript - guarantee above stays true by construction. - """ - - field: str - entity: str - requires: frozenset[str] - reads: frozenset[str] - reads_optional: frozenset[str] - check: EntityCheck - - -_DOCUMENT_RULES: Final[dict[str, list[Rule]]] = defaultdict(list) -_ENTITY_RULES: Final[dict[tuple[str, str], list[EntityRule]]] = defaultdict(list) -_DOCUMENT_KEYS: Final[dict[str, frozenset[str]]] = {} -_DISPATCHED_FIELDS: Final[set[str]] = set() - - -def register_document_type( - document_type: str, - standard_keys: frozenset[str], - extension_keys: frozenset[str] = frozenset(), -) -> None: - """Declare a document type's known keys, so `requires` can be checked. - - `extension_keys` names keys that are not part of the document's - TypedDict but that this package nonetheless recognizes — the v3 - `consolidated_metadata` convention is the only one today. Requiring - them to be declared here rather than exempting unknown keys wholesale - keeps the typo check meaningful. - """ - _DOCUMENT_KEYS[document_type] = standard_keys | extension_keys - - -def _validate_requires(document_type: str, requires: frozenset[str], what: str) -> None: - known = _DOCUMENT_KEYS.get(document_type) - if known is None: - msg = f"unknown document type {document_type!r} registering {what}" - raise LookupError(msg) - unknown = requires - known - if len(unknown) != 0: - msg = ( - f"{what} requires {sorted(unknown)}, which {document_type} documents " - f"do not have; such a rule could never fire" - ) - raise ValueError(msg) - - -def document_rule( - document_type: str, requires: frozenset[str] -) -> Callable[[Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]], Rule]: - """Register a whole-document rule, returning the `Rule` it becomes. - - The decorated function is replaced by its `Rule`, so a rule cannot be - defined without being registered, and referencing one by name yields - the registered object rather than a copy. - """ - - def decorate( - check: Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]], - ) -> Rule: - _validate_requires(document_type, requires, f"rule {check.__name__!r}") - rule = Rule(requires=requires, check=check) - _DOCUMENT_RULES[document_type].append(rule) - return rule - - return decorate - - -def entity_rule( - document_type: str, - field: ExtensionPointField, - entity: str, - requires: frozenset[str] = frozenset(), - reads: frozenset[str] = frozenset(), - reads_optional: frozenset[str] = frozenset(), -) -> Callable[[EntityCheck], EntityRule]: - """Register a rule about one named entity within `document_type`. - - The entity must already be shape-modelled in `zarr_metadata.v3._shape`: - entity rules read configuration members by name, so they only run once - the shape validator vouches those members exist and are typed. A rule - registered for an unmodelled name would silently never fire, so that - is refused here rather than discovered as a missing check later. - """ - - def decorate(check: EntityCheck) -> EntityRule: - _validate_requires(document_type, requires, f"entity rule {check.__name__!r}") - canonical_entity = canonical_name(field, entity) - if (field, canonical_entity) not in modelled_entities(): - msg = ( - f"entity rule {check.__name__!r} targets {entity!r}, which has no shape " - f"validator in zarr_metadata.v3._shape; such a rule could never fire" - ) - raise ValueError(msg) - modelled = entity_configuration_keys(field, entity) or frozenset() - required = entity_required_configuration_keys(field, entity) or frozenset() - unmodelled = (reads | reads_optional) - modelled - if len(unmodelled) != 0: - msg = ( - f"entity rule {check.__name__!r} declares {sorted(unmodelled)}, which " - f"{entity!r} does not model; such a member can never carry a value to read" - ) - raise ValueError(msg) - optional = reads - required - if len(optional) != 0: - msg = ( - f"entity rule {check.__name__!r} declares reads={sorted(optional)}, which " - f"{entity!r} does not require; an absent optional member is reported by " - f"nothing, so subscripting it would raise out of a validator. Declare it as " - f"reads_optional and test for presence instead." - ) - raise ValueError(msg) - rule = EntityRule( - field=field, - entity=entity, - requires=requires, - reads=reads, - reads_optional=reads_optional, - check=check, - ) - _ENTITY_RULES[field, canonical_entity].append(rule) - return rule - - return decorate - - -def document_rules(document_type: str) -> tuple[Rule, ...]: - """Every rule registered for `document_type`, in definition order.""" - return tuple(_DOCUMENT_RULES[document_type]) - - -def dispatched_fields() -> frozenset[str]: - """Extension points that have a dispatcher, so their rules can run. - - An entity rule registered at a field with no dispatcher is accepted and - then never fires — the silent-pass failure this module exists to - prevent. Checking coverage at registration would depend on import - order, so `tests/rules/test_registry.py` asserts it instead. - """ - return frozenset(_DISPATCHED_FIELDS) - - -def registered_entities() -> frozenset[tuple[str, str]]: - """Every `(field, canonical name)` that has at least one registered rule.""" - return frozenset(_ENTITY_RULES) - - -def run_entity_rules( - field: ExtensionPointField, - value: object, - document: Mapping[str, object], - loc: tuple[str | int, ...], - incoming: ArrayParts | None = None, -) -> tuple[ValidationProblem, ...]: - """Run the rules registered for whatever entity `value` names. - - Declines silently when `value` names nothing known, when its shape is - broken in a way that makes its configuration uninterpretable (the - shape rule owns that complaint), or when a rule's required document - keys are absent. An `unknown_key` never declines — see - `zarr_metadata.v3._shape.blocking_problems`. - """ - name = entity_name(value) - if name is None: - return () - rules = _ENTITY_RULES.get((field, canonical_name(field, name))) - if rules is None or len(rules) == 0: - return () - verdict = validate_known_entity_metadata(field, value) - if verdict is None: - return () - blocking = blocking_problems(verdict) - # Only two locations mean there is no configuration to read: the entity - # itself, and `configuration` as a whole. Anything else is about one - # member — including `must_understand`, which is part of the envelope - # and says nothing about whether the configuration is readable. - if any(problem.loc in ((), ("configuration",)) for problem in blocking): - return () - unusable = frozenset( - str(problem.loc[1]) - for problem in blocking - if len(problem.loc) >= 2 and problem.loc[0] == "configuration" - ) - configuration = configuration_mapping(value) - if configuration is None: - return () - problems: list[ValidationProblem] = [] - for rule in rules: - if not rule.requires <= document.keys(): - continue - if len((rule.reads | rule.reads_optional) & unusable) != 0: - continue - for found in rule.check(configuration, document, incoming): - # A rule that reports at the entity itself (an empty loc) is - # judging the whole entity, not a member of its configuration — - # and a bare-string entity has no `configuration` node to point at. - base = (*loc, "configuration") if len(found.loc) != 0 else loc - problems.append(ValidationProblem((*base, *found.loc), found.message, found.kind)) - return tuple(problems) - - -def dispatch_field( - field: ExtensionPointField, -) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: - """A check that runs entity rules for the entity in `document[field]`.""" - - def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - return run_entity_rules(field, document[field], document, (field,)) - - _DISPATCHED_FIELDS.add(field) - check.__name__ = f"_dispatch_{field}_entity_rules" - return check - - -def dispatch_field_sequence( - field: ExtensionPointField, -) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: - """A check that runs entity rules for every entity in `document[field]`.""" - - def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - entries = document[field] - if not isinstance(entries, (list, tuple)): - return () - sequence = cast("tuple[object, ...]", entries) - return run_chain_rules(field, sequence, document, (field,), chain_initial_spec(document)) - - _DISPATCHED_FIELDS.add(field) - check.__name__ = f"_dispatch_{field}_entity_rules" - return check - - -def run_chain_rules( - field: ExtensionPointField, - codecs: Sequence[object], - document: Mapping[str, object], - loc: tuple[str | int, ...], - initial: ArrayParts | None, -) -> tuple[ValidationProblem, ...]: - """Run entity rules over a codec chain, propagating the array spec. - - Each codec's rules receive the spec that codec actually receives — - the array as transformed by everything before it. Shared by the - top-level `codecs` dispatcher and by sharding, whose inner pipelines - are chains that start from the inner chunk. - """ - problems: list[ValidationProblem] = [] - for index, entry, incoming in propagate( - codecs, initial, lambda codec: entity_configuration(field, codec) - ): - problems.extend(run_entity_rules(field, entry, document, (*loc, index), incoming)) - return tuple(problems) - - -def chain_initial_spec(document: Mapping[str, object]) -> ArrayParts | None: - """What enters a document's top-level codec chain. - - The parts a chunk pipeline encodes are the chunks of the document's - chunk grid. A `data_type` that is not a metadata field has been - rejected structurally already, but it costs only itself: the grid is - still readable, and the geometry rules should still report what they - can rather than making the reader fix one fault to discover the rest. - """ - data_type = document.get("data_type") - grid = ChunkGrid.of(document.get("chunk_grid"), document.get("shape")) - if entity_name(data_type) is None: - return ArrayParts(grid, None) - return ArrayParts(grid, cast("ZarrV3MetadataFieldJSON", data_type)) - - -__all__ = [ - "EntityCheck", - "EntityRule", - "chain_initial_spec", - "dispatched_fields", - "document_rule", - "document_rules", - "entity_rule", - "registered_entities", - "run_chain_rules", - "run_entity_rules", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py b/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py deleted file mode 100644 index 72ac5711a9..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_spec.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Array specifications and how a codec chain transforms them. - -Each array->array codec transforms the array it receives, so a codec's -configuration must be judged against the array that *reaches* it, not -against the document's top-level fields: `transpose` permutes the shape, -`cast_value` changes the data type, and a `sharding_indexed` codec that -follows either one sees the transformed array. - -`ArrayParts` is every part of an array a codec will be handed, together -with their element type; `propagate` walks a chain handing each codec what -reaches it. There is no half-populated value: a codec receives `None` once -this package can no longer say what it operates on. An unknown codec might -change anything, so everything after one receives `None`, and so does -everything after the array->bytes boundary, where there is no array left. - -Transitions are registered per array->array codec, next to that codec's -rules, via `spec_transition`. A modelled codec with no transition is -treated as unknown, so a forgotten transition fails closed. -""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Final - -from zarr_metadata.rules._chunk_grid import ChunkGrid # noqa: TC001 -from zarr_metadata.v3._extension_points import CODECS, canonical_name -from zarr_metadata.v3._shape import entity_name -from zarr_metadata.v3.codec.kind import codec_kind_of_name - -if TYPE_CHECKING: - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - - -@dataclass(frozen=True, slots=True) -class ArrayParts: - """Every part of an array a codec will be handed, and their type. - - The parts an array is divided into, not the fields of its metadata. - Plural deliberately: one pipeline encodes every chunk, so a rule about - it quantifies over all of them — a shard's inner chunk shape must - divide *every* chunk, which under a rectilinear grid is several - different lengths. - - `data_type` is the metadata-field value verbatim, because rules compare - it by name, and it is `None` where the element type is undetermined - while the array itself is not. That happens inside a shard: the inner - grid is the sharding codec's own `chunk_shape` whatever reached it, so - an unreadable codec upstream costs the type and not the parts. `None` - in place of the whole value means something else again — that there is - no array here at all, past the array->bytes boundary or beyond a codec - that could have changed anything. - """ - - grid: ChunkGrid - data_type: ZarrV3MetadataFieldJSON | None - - def with_grid(self, grid: ChunkGrid) -> ArrayParts: - return replace(self, grid=grid) - - def with_data_type(self, data_type: ZarrV3MetadataFieldJSON | None) -> ArrayParts: - return replace(self, data_type=data_type) - - -SpecTransition = Callable[[Mapping[str, object], "ArrayParts"], "ArrayParts | None"] -"""How one codec transforms what it receives. - -Takes the codec's (shape-valid) configuration and the incoming parts, and -returns what the next codec sees, or `None` when this codec leaves nothing -determinable. A transition must never raise on the values the shape -validator admits. -""" - -_TRANSITIONS: Final[dict[str, SpecTransition]] = {} - - -def spec_transition(codec: str) -> Callable[[SpecTransition], SpecTransition]: - """Register how `codec` transforms the `ArrayParts` it receives. - - Only array->array codecs need one: array->bytes and bytes->bytes - codecs end shape propagation by construction, so registering a - transition for one is refused. - """ - kind = codec_kind_of_name(codec) - if kind != "array_array": - msg = ( - f"spec transition registered for {codec!r}, which is " - f"{kind or 'unknown'} rather than array_array; only array->array " - "codecs transform the array spec" - ) - raise ValueError(msg) - - def decorate(transition: SpecTransition) -> SpecTransition: - _TRANSITIONS[canonical_name(CODECS, codec)] = transition - return transition - - return decorate - - -def transitions_registered() -> frozenset[str]: - """Every codec name with a registered spec transition.""" - return frozenset(_TRANSITIONS) - - -def propagate( - codecs: Sequence[object], - initial: ArrayParts | None, - configuration_of: Callable[[object], Mapping[str, object] | None], -) -> Iterator[tuple[int, object, ArrayParts | None]]: - """Yield `(index, codec, incoming)` for each codec in the chain. - - `incoming` is `None` once propagation has stopped: after an unknown - codec, after a known codec whose configuration is not shape-valid, - after a codec this package has no transition for, and after the - array->bytes boundary, where there is no array to describe. - `configuration_of` resolves a codec entry to its usable configuration - (`entity_configuration` in practice; injected to keep this module free - of the registry). - """ - parts = initial - for index, codec in enumerate(codecs): - yield index, codec, parts - if parts is None: - continue - name = entity_name(codec) - kind = codec_kind_of_name(name) if name is not None else None - if kind == "array_array": - transition = _TRANSITIONS.get(canonical_name(CODECS, name or "")) - configuration = configuration_of(codec) - parts = ( - None - if transition is None or configuration is None - else transition(configuration, parts) - ) - else: - # An unknown codec may change anything; array->bytes consumes - # the array; bytes->bytes never had one. - parts = None - - -__all__ = [ - "ArrayParts", - "SpecTransition", - "propagate", - "spec_transition", - "transitions_registered", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py b/packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py deleted file mode 100644 index 19f9e2119b..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_storage_class.py +++ /dev/null @@ -1,141 +0,0 @@ -"""How a data type lays out in bytes, and what that demands of a codec. - -Two rules in this package need the same fact about a data type: whether -one of its scalars occupies a fixed number of bytes, and if so whether -that number is one (no byte order to declare) or more (a byte order the -`bytes` codec must declare). - -The `bytes` codec spec makes `endian` "Required for data types for which -endianness is applicable... multi-byte data types, such as `uint16` and -`int32`, but not single-byte data types, such as `uint8` or `bool`", and -addresses fixed-size numeric types only. - -The `struct` spec then states its own constraints in those same terms -rather than inventing new ones: a field's data type must be one "whose -size in bytes is fixed and known at the time the array is opened", since -variable-length types "do not have a fixed encoded size"; and "When a -`struct` type contains multi-byte numeric fields, the `bytes` codec MUST -be configured with an explicit `endian` setting", while a struct -"composed entirely of single-byte fields... MAY omit the `endian` -configuration". - -So a struct's own storage class is the widest class among its fields, -recursively, and "valid as a struct field" is simply "not -variable-length". One classifier answers both. - -- https://zarr-specs.readthedocs.io/en/latest/v3/codecs/bytes/index.html -- https://github.com/zarr-developers/zarr-extensions/blob/main/data-types/struct/README.md -""" - -from __future__ import annotations - -from typing import Literal, cast - -from zarr_metadata.rules._engine import as_string_mapping -from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN -from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME - -StorageClass = Literal["single_byte", "multi_byte", "variable_length"] -"""How one scalar of a data type occupies bytes. - -`single_byte` and `multi_byte` are both fixed-size; they differ only in -whether a byte order applies. -""" - -_SINGLE_BYTE = frozenset({BOOL_DATA_TYPE_NAME, INT8_DATA_TYPE_NAME, UINT8_DATA_TYPE_NAME}) -_MULTI_BYTE = frozenset( - { - INT16_DATA_TYPE_NAME, - INT32_DATA_TYPE_NAME, - INT64_DATA_TYPE_NAME, - UINT16_DATA_TYPE_NAME, - UINT32_DATA_TYPE_NAME, - UINT64_DATA_TYPE_NAME, - FLOAT16_DATA_TYPE_NAME, - FLOAT32_DATA_TYPE_NAME, - FLOAT64_DATA_TYPE_NAME, - COMPLEX64_DATA_TYPE_NAME, - COMPLEX128_DATA_TYPE_NAME, - NUMPY_DATETIME64_DATA_TYPE_NAME, - NUMPY_TIMEDELTA64_DATA_TYPE_NAME, - } -) -_VARIABLE_LENGTH = frozenset({BYTES_DATA_TYPE_NAME, STRING_DATA_TYPE_NAME}) - - -def data_type_name(data_type: object) -> str | None: - """The name a data-type metadata field carries, or None if it has none.""" - if isinstance(data_type, str): - return data_type - mapping = as_string_mapping(data_type) - if mapping is None: - return None - name = mapping.get("name") - return name if isinstance(name, str) else None - - -def storage_class(data_type: object) -> StorageClass | None: - """Classify a known data type by its raw byte representation. - - None means undetermined — an unknown name, or a `struct` whose fields - this package cannot read — and every rule declines rather than - guessing. A `struct` takes the widest class among its fields, so a - struct of `uint8` and `int32` is `multi_byte` and one containing a - `string` is `variable_length`, recursively. - """ - name = data_type_name(data_type) - if name in _SINGLE_BYTE or (name is not None and RAW_BYTES_NAME_PATTERN.fullmatch(name)): - return "single_byte" - if name in _MULTI_BYTE: - return "multi_byte" - if name in _VARIABLE_LENGTH: - return "variable_length" - if name != STRUCT_DATA_TYPE_NAME: - return None - - envelope = as_string_mapping(data_type) - configuration = ( - as_string_mapping(envelope.get("configuration")) if envelope is not None else None - ) - fields = configuration.get("fields") if configuration is not None else None - if not isinstance(fields, tuple): - return None - classes: set[StorageClass] = set() - for field in cast("tuple[object, ...]", fields): - field_mapping = as_string_mapping(field) - if field_mapping is None or "data_type" not in field_mapping: - return None - field_class = storage_class(field_mapping["data_type"]) - if field_class is None: - return None - classes.add(field_class) - if "variable_length" in classes: - return "variable_length" - if "multi_byte" in classes: - return "multi_byte" - return "single_byte" - - -__all__ = [ - "StorageClass", - "data_type_name", - "storage_class", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py index cd4493dc59..376eae259e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py @@ -1,38 +1,34 @@ -"""Composition rules for v2 array metadata documents. +"""Semantic checks for v2 array metadata documents. -The v2 rule set is deliberately small today: the one cross-field -constraint the package interprets is that `chunks` and `shape` agree on -dimensionality. Fill-value/dtype consistency for v2 (NumPy dtype strings, -base64 fills for bytes dtypes) is a known follow-up, tracked in the -package docs. +Deliberately small: the one cross-field constraint the package interprets +is that `chunks` and `shape` agree on dimensionality. v2 has no extension +mechanism, so there are no entities to ask -- this is the whole of it. +Fill-value/dtype consistency for v2 (NumPy dtype strings, base64 fills +for bytes dtypes) is a known follow-up, tracked in the package docs. """ from __future__ import annotations -from typing import TYPE_CHECKING, Final +from collections.abc import Sequence +from typing import TYPE_CHECKING, cast -from zarr_metadata.model._validation import ( - ARRAY_METADATA_STANDARD_KEYS_V2, - ValidationProblem, -) -from zarr_metadata.rules._engine import Rule, as_sequence -from zarr_metadata.rules._registry import document_rule, document_rules, register_document_type +from zarr_metadata.model._validation import ValidationProblem if TYPE_CHECKING: from collections.abc import Mapping -ZARR_V2_ARRAY = "zarr_v2_array" -"""Document-type key under which this module's rules are registered.""" +def _as_sequence(value: object) -> tuple[object, ...] | None: + """`value` as a tuple if it is a JSON array, else None.""" + if isinstance(value, str) or not isinstance(value, Sequence): + return None + return tuple(cast("Sequence[object]", value)) -register_document_type(ZARR_V2_ARRAY, ARRAY_METADATA_STANDARD_KEYS_V2) - -@document_rule(ZARR_V2_ARRAY, frozenset({"shape", "chunks"})) -def check_chunks_match_shape(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """`chunks` must have one entry per dimension of `shape`.""" - shape = as_sequence(document["shape"]) - chunks = as_sequence(document["chunks"]) +def array_problems_v2(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Every semantic problem in a v2 array document.""" + shape = _as_sequence(document.get("shape")) + chunks = _as_sequence(document.get("chunks")) if shape is None or chunks is None or len(shape) == len(chunks): return () return ( @@ -44,11 +40,6 @@ def check_chunks_match_shape(document: Mapping[str, object]) -> tuple[Validation ) -ZARR_V2_ARRAY_RULES: Final[tuple[Rule, ...]] = document_rules(ZARR_V2_ARRAY) -"""The composition rule set for v2 array metadata documents.""" - - __all__ = [ - "ZARR_V2_ARRAY", - "ZARR_V2_ARRAY_RULES", + "array_problems_v2", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py deleted file mode 100644 index 8df9b1c398..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_array.py +++ /dev/null @@ -1,448 +0,0 @@ -"""Rules over a whole v3 array metadata document. - -The same three-way split the entity layer uses applies here, one level -up. **Type** is the document's structure, checked by -`zarr_metadata.model`. **Value** is a constraint on one field's own -content — the codec pipeline's kind ordering, a known entity's canonical -shape — which needs nothing else in the document to decide. **Composition** -is a judgment spanning fields: a fill value against its data type, one -dimension name per dimension of `shape`. - -A rule's `requires` says which it is, and the two are kept in separate -sections below; `tests/rules/test_registry.py` asserts the partition, so -adding a rule is a deliberate choice rather than an accident of where the -cursor was. Value rules over a single *entity* go one level further down, -into `v3._shape` if they constrain a member and into `rules._entities` if -they constrain elements within one. - -Whole-document rules live here: judgments that read several top-level -fields, or that apply to a field regardless of which extension occupies -it. Rules about a *particular* codec or chunk grid live with that entity -in `zarr_metadata.rules._entities`, registered by name — so adding a -third chunk grid or a new codec adds a module there and changes nothing -in this one. The `codecs` and `chunk_grid` dispatchers below are generic: -they run whatever rules are registered for the name they find. - -Extension openness: rules never reject what they cannot interpret. An -unknown data type name accepts any fill value here (its own validator is -whoever understands it), an unknown codec has unknown kind, and unknown -entities pass through untouched. Openness is for genuinely unknown names -only: a codec or chunk-grid name this package defines is held to its full -canonical shape (via `zarr_metadata.v3._shape`), and a known codec ranks -as its pipeline kind in every spelling — otherwise a misspelled known -name would masquerade as an unknown extension and silently escape both -the shape and the ordering checks. -""" - -from __future__ import annotations - -import importlib -from collections.abc import Mapping -from typing import TYPE_CHECKING, Final, cast - -from zarr_metadata.model._validation import ( - ARRAY_METADATA_STANDARD_KEYS_V3, - ValidationProblem, -) -from zarr_metadata.rules._engine import Rule, as_sequence, as_string_mapping, prefixed -from zarr_metadata.rules._pipeline import pipeline_order_problems, shape_problems -from zarr_metadata.rules._registry import ( - dispatch_field, - dispatch_field_sequence, - document_rule, - document_rules, - register_document_type, -) -from zarr_metadata.v3._extension_points import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, - ExtensionPointField, -) -from zarr_metadata.v3._shape import ( - validate_known_chunk_grid_metadata, - validate_known_entity_metadata, -) -from zarr_metadata.v3.data_type.bytes import base64_bytes -from zarr_metadata.v3.data_type.float16 import hex_float16 -from zarr_metadata.v3.data_type.float32 import hex_float32 -from zarr_metadata.v3.data_type.float64 import hex_float64 -from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN, raw_bytes_dtype_name - -if TYPE_CHECKING: - from collections.abc import Callable - - -# --------------------------------------------------------------------------- -# fill_value vs. data_type -# --------------------------------------------------------------------------- - -_INT_RANGES: Final[dict[str, tuple[int, int]]] = { - "int8": (-(2**7), 2**7 - 1), - "int16": (-(2**15), 2**15 - 1), - "int32": (-(2**31), 2**31 - 1), - "int64": (-(2**63), 2**63 - 1), - "uint8": (0, 2**8 - 1), - "uint16": (0, 2**16 - 1), - "uint32": (0, 2**32 - 1), - "uint64": (0, 2**64 - 1), -} - -_FLOAT_HEX_VALIDATORS: Final[dict[str, Callable[[str], object]]] = { - "float16": hex_float16, - "float32": hex_float32, - "float64": hex_float64, -} - -_COMPLEX_COMPONENT_TYPES: Final[dict[str, str]] = { - "complex64": "float32", - "complex128": "float64", -} - -_FLOAT_SPECIALS: Final = frozenset({"NaN", "Infinity", "-Infinity"}) - - -def _is_int(value: object) -> bool: - # bool is an int subtype but is never a valid integer fill value. - return isinstance(value, int) and not isinstance(value, bool) - - -def _check_float_fill(value: object, dtype_name: str) -> str | None: - if _is_int(value) or isinstance(value, float): - return None - if isinstance(value, str): - if value in _FLOAT_SPECIALS: - return None - try: - _FLOAT_HEX_VALIDATORS[dtype_name](value) - except ValueError: - return ( - f"expected a number, one of 'NaN'/'Infinity'/'-Infinity', or a " - f"{dtype_name} hex string, got {value!r}" - ) - return None - return f"expected a number or string, got {value!r}" - - -def _check_byte_sequence(value: object, expected_len: int | None) -> str | None: - items = as_sequence(value) - if items is None: - return f"expected an array of byte values, got {value!r}" - if expected_len is not None and len(items) != expected_len: - return f"expected {expected_len} byte values, got {len(items)}" - for item in items: - if not _is_int(item) or not 0 <= cast(int, item) <= 255: - return f"expected integers in [0, 255], got {item!r}" - return None - - -def _check_fill_for_dtype(dtype_name: str, value: object) -> str | None: - """Why `value` is not a valid fill value for `dtype_name`, or None. - - Unknown data type names accept anything (extension openness). - """ - if dtype_name == "bool": - return None if isinstance(value, bool) else f"expected a boolean, got {value!r}" - if dtype_name in _INT_RANGES: - low, high = _INT_RANGES[dtype_name] - if not _is_int(value): - return f"expected an integer, got {value!r}" - if not low <= cast(int, value) <= high: - return f"expected an integer in [{low}, {high}], got {value!r}" - return None - if dtype_name in _FLOAT_HEX_VALIDATORS: - return _check_float_fill(value, dtype_name) - if dtype_name in _COMPLEX_COMPONENT_TYPES: - component = _COMPLEX_COMPONENT_TYPES[dtype_name] - pair = as_sequence(value) - if pair is None or len(pair) != 2: - return f"expected a [real, imag] pair, got {value!r}" - for part in pair: - reason = _check_float_fill(part, component) - if reason is not None: - return f"invalid component: {reason}" - return None - if dtype_name == "string": - return None if isinstance(value, str) else f"expected a string, got {value!r}" - if dtype_name == "bytes": - if isinstance(value, str): - try: - base64_bytes(value) - except ValueError: - return f"expected standard-alphabet base64, got {value!r}" - return None - return _check_byte_sequence(value, None) - if dtype_name in ("numpy.datetime64", "numpy.timedelta64"): - if value == "NaT": - return None - if not _is_int(value): - return f"expected a signed 64-bit integer or 'NaT', got {value!r}" - if not -(2**63) <= cast(int, value) <= 2**63 - 1: - return f"expected a signed 64-bit integer, got {value!r}" - return None - if dtype_name == "struct": - if isinstance(value, Mapping): - return None - return f"expected an object of per-field fill values, got {value!r}" - if RAW_BYTES_NAME_PATTERN.fullmatch(dtype_name) is not None: - try: - raw_bytes_dtype_name(dtype_name) - except ValueError: - return None # malformed r name: _check_data_type_spelling reports it - return _check_byte_sequence(value, int(dtype_name[1:]) // 8) - return None # unknown data type: its fill values are not ours to judge - - -def _dtype_name(data_type: object) -> str | None: - if isinstance(data_type, str): - return data_type - mapping = as_string_mapping(data_type) - if mapping is not None: - name = mapping.get("name") - if isinstance(name, str): - return name - return None # structurally invalid; the structural validator reports it - - -def _check_data_type_spelling(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """Misspellings of data type families this package defines. - - An `r` name whose bit count is not a positive multiple of 8 is a - misspelling of the known raw-bytes family, not an unknown extension: - treating it as unknown would let the misspelling masquerade as an - extension and escape judgment entirely (the same anti-masquerade - reasoning as the codec spelling checks). Genuinely unknown names pass - untouched. - """ - name = _dtype_name(document["data_type"]) - if name is None or RAW_BYTES_NAME_PATTERN.fullmatch(name) is None: - return () - try: - raw_bytes_dtype_name(name) - except ValueError as error: - return (ValidationProblem(("data_type",), str(error), "invalid_value"),) - return () - - -def _check_fill_matches_dtype(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - data_type = document["data_type"] - dtype_name = _dtype_name(data_type) - if dtype_name is None: - return () - if dtype_name == "struct": - return _struct_fill_problems(data_type, document["fill_value"], ("fill_value",)) - reason = _check_fill_for_dtype(dtype_name, document["fill_value"]) - if reason is None: - return () - return ( - ValidationProblem( - ("fill_value",), - f"fill_value invalid for data_type {dtype_name!r}: {reason}", - "invalid_value", - ), - ) - - -def _struct_fill_problems( - data_type: object, fill_value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - """Validate a struct fill mapping against every field, recursively.""" - if not isinstance(fill_value, Mapping): - return ( - ValidationProblem( - loc, - f"fill_value invalid for data_type 'struct': expected an object of " - f"per-field fill values, got {fill_value!r}", - "invalid_value", - ), - ) - envelope = as_string_mapping(data_type) - configuration = ( - as_string_mapping(envelope.get("configuration")) if envelope is not None else None - ) - fields = configuration.get("fields") if configuration is not None else None - if not isinstance(fields, tuple): - return () # malformed data type: the structural validator owns it - - fill_mapping = cast("Mapping[object, object]", fill_value) - problems: list[ValidationProblem] = [] - field_names: set[str] = set() - for field in cast("tuple[object, ...]", fields): - field_mapping = as_string_mapping(field) - if field_mapping is None: - continue - name = field_mapping.get("name") - field_data_type = field_mapping.get("data_type") - if not isinstance(name, str) or field_data_type is None: - continue - field_names.add(name) - field_loc = (*loc, name) - if name not in fill_mapping: - problems.append( - ValidationProblem( - field_loc, f"missing fill value for struct field {name!r}", "missing_key" - ) - ) - continue - field_fill = fill_mapping[name] - nested_name = _dtype_name(field_data_type) - if nested_name == "struct": - problems.extend(_struct_fill_problems(field_data_type, field_fill, field_loc)) - continue - if nested_name is None: - continue - reason = _check_fill_for_dtype(nested_name, field_fill) - if reason is not None: - problems.append( - ValidationProblem( - field_loc, - f"fill value invalid for struct field {name!r} with data_type " - f"{nested_name!r}: {reason}", - "invalid_value", - ) - ) - problems.extend( - ValidationProblem((*loc, key), f"unknown struct fill field {key!r}", "unknown_key") - for key in sorted( - candidate - for candidate in fill_mapping.keys() - field_names - if isinstance(candidate, str) - ) - ) - return tuple(problems) - - -# --------------------------------------------------------------------------- -# field rules: each reads one top-level field and nothing else -# --------------------------------------------------------------------------- - -ZARR_V3_ARRAY = "zarr_v3_array" -"""Document-type key under which this module's rules are registered.""" - -register_document_type(ZARR_V3_ARRAY, ARRAY_METADATA_STANDARD_KEYS_V3) - -_data_type_spelling = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( - _check_data_type_spelling -) - - -def _known_entity_shape( - field: ExtensionPointField, -) -> Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]]: - """A check that judges `document[field]` against `field`'s known shapes. - - One parameter, not two: the document field and the extension point are - the same thing, and taking them separately invited passing a codec - under the chunk-grid point. - """ - - def check(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - found = validate_known_entity_metadata(field, document[field]) - return () if found is None else prefixed((field,), found) - - check.__name__ = f"_check_{field}_shape" - return check - - -_data_type_shape = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( - _known_entity_shape(DATA_TYPE) -) -_chunk_key_encoding_shape = document_rule(ZARR_V3_ARRAY, frozenset({"chunk_key_encoding"}))( - _known_entity_shape(CHUNK_KEY_ENCODING) -) - - -@document_rule(ZARR_V3_ARRAY, frozenset({"codecs"})) -def check_codec_pipeline_order(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """The pipeline shape: `array->array`* `array->bytes` `bytes->bytes`*.""" - entries = as_sequence(document["codecs"]) - if entries is None: - return () - return pipeline_order_problems(entries, ("codecs",)) - - -@document_rule(ZARR_V3_ARRAY, frozenset({"codecs"})) -def check_codec_shapes(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """Every known-name codec matches its canonical type.""" - entries = as_sequence(document["codecs"]) - if entries is None: - return () - return shape_problems(entries, ("codecs",)) - - -@document_rule(ZARR_V3_ARRAY, frozenset({"chunk_grid"})) -def check_chunk_grid_shape(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """A known-name chunk grid matches its canonical type.""" - found = validate_known_chunk_grid_metadata(document["chunk_grid"]) - # None is "not a known grid" (unjudged); () is "known and valid". - if found is None: - return () - return prefixed(("chunk_grid",), found) - - -# --------------------------------------------------------------------------- -# composition rules: each spans more than one top-level field -# --------------------------------------------------------------------------- - -_fill_matches_dtype = document_rule(ZARR_V3_ARRAY, frozenset({"data_type", "fill_value"}))( - _check_fill_matches_dtype -) - - -@document_rule(ZARR_V3_ARRAY, frozenset({"shape", "dimension_names"})) -def check_dimension_names_length(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """One dimension name per array dimension.""" - shape = as_sequence(document["shape"]) - names = as_sequence(document["dimension_names"]) - if shape is None or names is None or len(names) == len(shape): - return () - return ( - ValidationProblem( - ("dimension_names",), - f"dimension_names has {len(names)} entries but shape has {len(shape)} dimensions", - "invalid_value", - ), - ) - - -# Generic dispatchers: every rule an entity registers for itself runs here, -# so a new codec, chunk grid, data type, or chunk key encoding needs no edit -# to this module. There must be one per extension point that has shapes: -# without it, `entity_rule` accepts a registration whose rule can never run, -# which is the silent-pass failure the registry exists to prevent. -# `test_registry.py` asserts that coverage. -_dispatch_chunk_grid = document_rule(ZARR_V3_ARRAY, frozenset({"chunk_grid"}))( - dispatch_field(CHUNK_GRID) -) -_dispatch_data_type = document_rule(ZARR_V3_ARRAY, frozenset({"data_type"}))( - dispatch_field(DATA_TYPE) -) -_dispatch_chunk_key_encoding = document_rule(ZARR_V3_ARRAY, frozenset({"chunk_key_encoding"}))( - dispatch_field(CHUNK_KEY_ENCODING) -) -_dispatch_codecs = document_rule(ZARR_V3_ARRAY, frozenset({"codecs"}))( - dispatch_field_sequence(CODECS) -) - - -def _rules() -> tuple[Rule, ...]: - # Importing the entity package registers every entity's rules; done here - # rather than at module import to keep the dependency one-directional. - importlib.import_module("zarr_metadata.rules._entities") - - return document_rules(ZARR_V3_ARRAY) - - -ZARR_V3_ARRAY_RULES: Final[tuple[Rule, ...]] = _rules() -"""The composition rule set for v3 array metadata documents. - -Assembled from the registry rather than written out, so a rule cannot be -defined without joining the set it belongs to. -""" - - -__all__ = [ - "ZARR_V3_ARRAY", - "ZARR_V3_ARRAY_RULES", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py index 5023ef11bd..99b6f39166 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py @@ -1,85 +1,84 @@ -"""Composition rules for v3 group metadata documents. - -A group document's own fields carry no cross-field constraints, but the -inline consolidated-metadata convention embeds whole child documents — -and a composition-invalid child makes the consolidated view lie about -the store. The group rule set therefore recurses: every array entry is -judged by the v3 array rules, and every group entry (which may itself -carry consolidated metadata) by this rule set. +"""Semantic checks for v3 group metadata documents. + +A group says almost nothing that can be wrong on its own. The one thing +it can carry is consolidated metadata -- the child documents of a whole +subtree, inline -- and each of those is judged exactly as it would be +standing alone, at its own path. + +`consolidated_metadata` is not a declared member of the group TypedDict: +the spec grandfathers it as a convention that "lacks the name member +required of extension objects". """ from __future__ import annotations -from typing import TYPE_CHECKING, Final +from collections.abc import Mapping +from typing import TYPE_CHECKING, cast -from zarr_metadata.model._validation import GROUP_METADATA_STANDARD_KEYS_V3 -from zarr_metadata.rules._engine import Rule, as_string_mapping, prefixed, run_rules -from zarr_metadata.rules._registry import document_rule, document_rules, register_document_type -from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY_RULES -from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY +from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._document import array_problems_v3 +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Sequence - from zarr_metadata.model._validation import ValidationProblem + from zarr_metadata.v3._entity import Loc -ZARR_V3_GROUP = "zarr_v3_group" -"""Document-type key under which this module's rules are registered.""" +def _prefixed(loc: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: + """Re-base every problem's `loc` under `loc`, for a nested document.""" + return tuple( + ValidationProblem((*loc, *found.loc), found.message, found.kind) for found in problems + ) -# `consolidated_metadata` is not a declared member of the group TypedDict: -# the spec grandfathers it as a convention that 'lacks the name member -# required of extension objects'. It is declared here so the rule that -# reads it passes the typo check without exempting unknown keys. -register_document_type( - ZARR_V3_GROUP, - GROUP_METADATA_STANDARD_KEYS_V3, - extension_keys=frozenset({ZARR_V3_CONSOLIDATED_METADATA_KEY}), -) +def _as_string_mapping(value: object) -> Mapping[str, object] | None: + """`value` as a string-keyed mapping, or None if it is not one.""" + if not isinstance(value, Mapping): + return None + mapping = cast("Mapping[object, object]", value) + if any(not isinstance(key, str) for key in mapping): + return None + return cast("Mapping[str, object]", mapping) -@document_rule(ZARR_V3_GROUP, frozenset({"consolidated_metadata"})) -def check_consolidated_entries(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """Consolidated child documents must satisfy their own composition rules. - Structural validity of the consolidated envelope and its entries is - the model layer's job; entries that are not interpretable as node - documents decline in its favor. - """ +def group_problems_v3(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Every semantic problem in a v3 group document.""" + if "consolidated_metadata" not in document: + return () return consolidated_entries_problems( document["consolidated_metadata"], ("consolidated_metadata",) ) -def consolidated_entries_problems( - value: object, loc: tuple[str | int, ...] = () -) -> tuple[ValidationProblem, ...]: - """Composition problems in an inline consolidated envelope's children.""" - consolidated = as_string_mapping(value) +def consolidated_entries_problems(value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + """Semantic problems in an inline consolidated envelope's children. + + Structural validity of the envelope and its entries is the model + layer's job; an entry that is not interpretable as a node document + declines in its favour. + """ + consolidated = _as_string_mapping(value) if consolidated is None: return () - metadata = as_string_mapping(consolidated.get("metadata")) + metadata = _as_string_mapping(consolidated.get("metadata")) if metadata is None: return () problems: list[ValidationProblem] = [] for path, entry in metadata.items(): - node = as_string_mapping(entry) + node = _as_string_mapping(entry) if node is None: continue entry_loc = (*loc, "metadata", path) node_type = node.get("node_type") if node_type == "array": - problems.extend(prefixed(entry_loc, run_rules(ZARR_V3_ARRAY_RULES, node))) + problems.extend(_prefixed(entry_loc, array_problems_v3(node, CORE_AND_EXTENSIONS))) elif node_type == "group": - problems.extend(prefixed(entry_loc, run_rules(ZARR_V3_GROUP_RULES, node))) + problems.extend(_prefixed(entry_loc, group_problems_v3(node))) return tuple(problems) -ZARR_V3_GROUP_RULES: Final[tuple[Rule, ...]] = document_rules(ZARR_V3_GROUP) -"""The composition rule set for v3 group metadata documents.""" - - __all__ = [ - "ZARR_V3_GROUP", - "ZARR_V3_GROUP_RULES", + "consolidated_entries_problems", + "group_problems_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py deleted file mode 100644 index 8a94ab7981..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py +++ /dev/null @@ -1,806 +0,0 @@ -""" -Type-level shape validation for known metadata-field entities. - -One validator per extension-point entity this package defines, exact with -respect to the entity's declared TypedDicts: a value yields no problems -exactly when it is an instance of the canonical metadata type, so a -verdict here means the same thing the type does. - -Two package-wide conventions qualify "exact": - -- `int`-annotated fields mean JSON integers, so JSON booleans are - rejected even though `bool` is an `int` subtype in Python's type - system (matching the fill-value rules' treatment of integers). -- Judgments are at the canonical data level: JSON arrays are tuples, as - the TypedDicts declare. Normalize a freshly-`json.loads`-ed document - (e.g. with a model-layer parser) before asking for shape verdicts. - -Three kinds of judgment, and this module owns the first two: - -- **type**: is this an integer? — the TypedDicts, checked member by member. -- **value**: is it an integer *in [0, 9]*? — a constraint the spec places on - one member's value, or on one entity's own configuration. Stated here as - a richer checker, or as an entity `invariant` when it spans two members - of the same configuration, because it is a refinement of the type and - needs nothing outside the entity to decide. -- **composition**: does this codec's rank match the array that reached it? - — needs the document or the codec chain, and belongs to - `zarr_metadata.rules`. - -Putting value constraints here rather than in the rule layer keeps the -answer next to the type it refines, and means a rule only exists where a -judgment genuinely spans more than one entity. - -The boundary is the *member*, and it is not a matter of taste. A verdict -here marks a whole member unusable, so a constraint on the elements -*within* one — every extent of a `chunk_shape` being positive — would cost -precision elsewhere: a zero on one axis would stop a rectilinear grid -reporting the axis beside it, and would stand down a shard's inner -pipeline entirely. Those stay in the rule layer, which judges element by -element. Constraints on a member as a whole belong here. - -Unknown names are not judged (extension openness): the `validate_known_*` -functions answer `None` for entities this package has no types for, no -problems for a valid known entity, and problems otherwise. - -Key sets are derived from the TypedDicts' `__annotations__` / -`__required_keys__` rather than restated by hand, so those entries cannot -drift from the canonical types; only the per-field value checks are -written out. The exception is `_BARE_DATA_TYPE_NAMES`: the core scalar -data types have no TypedDict to derive from — their whole metadata is a -name — so that list is hand-written, and `tests/test_registry_drift.py` -ties it to the modules that define those names. -""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass -from typing import TYPE_CHECKING, Final, cast - -from zarr_metadata.model._validation import ( - ValidationProblem, - is_json, - is_metadata_field_v3, -) -from zarr_metadata.v3._extension_points import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, - RAW_BYTES_FAMILY, - ExtensionPointField, - canonical_name, -) -from zarr_metadata.v3.chunk_grid.rectilinear import ( - RECTILINEAR_CHUNK_GRID_NAME, - RectilinearChunkGridConfiguration, - RectilinearChunkGridObject, -) -from zarr_metadata.v3.chunk_grid.regular import ( - REGULAR_CHUNK_GRID_NAME, - RegularChunkGridConfiguration, - RegularChunkGridObject, -) -from zarr_metadata.v3.chunk_key_encoding.default import ( - DEFAULT_CHUNK_KEY_ENCODING_NAME, - DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, - DefaultChunkKeyEncodingConfiguration, - DefaultChunkKeyEncodingObject, -) -from zarr_metadata.v3.chunk_key_encoding.v2 import ( - V2_CHUNK_KEY_ENCODING_NAME, - V2_CHUNK_KEY_ENCODING_SEPARATOR, - V2ChunkKeyEncodingConfiguration, - V2ChunkKeyEncodingObject, -) -from zarr_metadata.v3.codec.blosc import ( - BLOSC_CNAME, - BLOSC_CODEC_NAME, - BLOSC_NO_SHUFFLE, - BLOSC_SHUFFLE, - BloscCodecConfiguration, - BloscCodecObject, -) -from zarr_metadata.v3.codec.bytes import ( - BYTES_CODEC_NAME, - ENDIANNESS, - BytesCodecConfiguration, - BytesCodecObject, -) -from zarr_metadata.v3.codec.cast_value import ( - CAST_OUT_OF_RANGE_MODE, - CAST_ROUNDING_MODE, - CAST_VALUE_CODEC_NAME, - CastValueCodecConfiguration, - CastValueCodecObject, - ScalarMap, -) -from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME, Crc32cCodecObject, Empty -from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME, GzipCodecConfiguration, GzipCodecObject -from zarr_metadata.v3.codec.scale_offset import ( - SCALE_OFFSET_CODEC_NAME, - ScaleOffsetCodecConfiguration, - ScaleOffsetCodecObject, -) -from zarr_metadata.v3.codec.sharding_indexed import ( - SHARDING_INDEX_LOCATION, - SHARDING_INDEXED_CODEC_NAME, - ShardingIndexedCodecConfiguration, - ShardingIndexedCodecObject, -) -from zarr_metadata.v3.codec.transpose import ( - TRANSPOSE_CODEC_NAME, - TransposeCodecConfiguration, - TransposeCodecObject, -) -from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecConfiguration, ZstdCodecObject -from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.numpy_datetime64 import ( - NUMPY_DATETIME64_DATA_TYPE_NAME, - NumpyDatetime64, - NumpyDatetime64Configuration, -) -from zarr_metadata.v3.data_type.numpy_timedelta64 import ( - NUMPY_TIME_UNIT, - NUMPY_TIMEDELTA64_DATA_TYPE_NAME, - NumpyTimedelta64, - NumpyTimedelta64Configuration, -) -from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.struct import ( - STRUCT_DATA_TYPE_NAME, - Struct, - StructConfiguration, - StructField, -) -from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE_NAME -from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME - -if TYPE_CHECKING: - from zarr_metadata.model._validation import ProblemKind - -_FieldChecker = Callable[[object, tuple[str | int, ...]], tuple[ValidationProblem, ...]] -"""A constraint on one configuration member's value, given its location.""" - -_EntityInvariant = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] -"""A value constraint spanning two members of one configuration. - -Runs only once every member has passed its own checker, so it may read -them without guarding; `blosc`'s "typesize is required unless shuffle is -noshuffle" is the whole population today. Locations are relative to the -configuration. -""" - - -def entity_name(value: object) -> str | None: - """The `name` of a metadata-field entry in any spelling, or None. - - A bare string is its own name; an object's name is its `name` member. - Anything else (or a mapping without a string `name`) has no name and - is not interpretable as a known entity. - """ - if isinstance(value, str): - return value - if not isinstance(value, Mapping): - return None - name = cast("Mapping[object, object]", value).get("name") - return name if isinstance(name, str) else None - - -def _problems( - loc: tuple[str | int, ...], message: str, kind: ProblemKind = "invalid_type" -) -> tuple[ValidationProblem, ...]: - return (ValidationProblem(loc, message, kind),) - - -def _check_json_int(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if isinstance(value, bool) or not isinstance(value, int): - return _problems(loc, f"expected an integer, got {value!r}") - return () - - -def _check_json_bool(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if not isinstance(value, bool): - return _problems(loc, f"expected a boolean, got {value!r}") - return () - - -def _int_in_range(low: int, high: int) -> _FieldChecker: - """An integer the spec confines to `[low, high]`.""" - - def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - problems = _check_json_int(value, loc) - if len(problems) != 0: - return problems - if low <= cast("int", value) <= high: - return () - return _problems( - loc, f"expected an integer in [{low}, {high}], got {value!r}", "invalid_value" - ) - - return check - - -def _bounded_int(low: int, description: str) -> _FieldChecker: - """An integer the spec bounds from below only.""" - - def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - problems = _check_json_int(value, loc) - if len(problems) != 0: - return problems - if cast("int", value) >= low: - return () - return _problems(loc, f"expected {description}, got {value!r}", "invalid_value") - - return check - - -_check_positive_int = _bounded_int(1, "a positive integer") -_check_non_negative_int = _bounded_int(0, "a non-negative integer") - - -def _check_permutation(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - """A transpose order: a permutation of its own indices. - - Whether it also matches the rank of the array that reached the codec - is composition, and lives in `rules._entities.transpose`. - """ - problems = _check_int_tuple(value, loc) - if len(problems) != 0: - return problems - order = cast("tuple[int, ...]", value) - if sorted(order) == list(range(len(order))): - return () - return _problems( - loc, f"expected a permutation of 0..{len(order) - 1}, got {order!r}", "invalid_value" - ) - - -def _blosc_typesize_is_present_when_shuffling( - configuration: Mapping[str, object], -) -> tuple[ValidationProblem, ...]: - """`typesize` is required unless `shuffle` is `"noshuffle"`. - - The one constraint in this package that spans two members of a single - configuration, and the reason `_EntityShape` carries invariants at all. - """ - shuffle = configuration.get("shuffle") - if shuffle == BLOSC_NO_SHUFFLE or "typesize" in configuration: - return () - return _problems( - ("typesize",), f"typesize is required when shuffle is {shuffle!r}", "missing_key" - ) - - -def _literal(allowed: tuple[str, ...]) -> _FieldChecker: - def check(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if value not in allowed: - return _problems(loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value") - return () - - return check - - -def _check_int_tuple(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple): - return _problems(loc, f"expected an array (tuple) of integers, got {value!r}") - items = cast("tuple[object, ...]", value) - return tuple( - problem - for index, item in enumerate(items) - for problem in _check_json_int(item, (*loc, index)) - ) - - -def _check_json_value(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if not is_json(value): - return _problems(loc, "expected a JSON value") - return () - - -def _check_metadata_field( - value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - if not is_metadata_field_v3(value): - return _problems(loc, "expected a metadata field (bare name or name/configuration object)") - return () - - -def _check_field_tuple(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple): - return _problems(loc, f"expected an array (tuple) of metadata fields, got {value!r}") - items = cast("tuple[object, ...]", value) - return tuple( - problem - for index, item in enumerate(items) - for problem in _check_metadata_field(item, (*loc, index)) - ) - - -_STRUCT_FIELD_KEYS: Final = frozenset(StructField.__annotations__) - - -def _check_data_type_field( - value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - """A nested `data_type` position: structural shape plus its known shape. - - `cast_value`'s target type and a struct field's type are data types - like any other, so they get the judgment a top-level `data_type` gets. - Without this the same value is accepted in one position and rejected - in another — a bare `"numpy.datetime64"` is invalid at the top level - (its configuration is required) and was silently fine inside a struct. - Recurses naturally: a struct of structs is judged all the way down. - """ - problems = _check_metadata_field(value, loc) - if len(problems) != 0: - return problems - found = validate_known_entity_metadata(DATA_TYPE, value) - return () if found is None else _prefixed_at(loc, found) - - -def _prefixed_at( - loc: tuple[str | int, ...], problems: tuple[ValidationProblem, ...] -) -> tuple[ValidationProblem, ...]: - return tuple( - ValidationProblem((*loc, *problem.loc), problem.message, problem.kind) - for problem in problems - ) - - -def _check_struct_fields( - value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple): - return _problems(loc, f"expected an array (tuple) of struct fields, got {value!r}") - problems: list[ValidationProblem] = [] - for index, item in enumerate(cast("tuple[object, ...]", value)): - item_loc = (*loc, index) - if not isinstance(item, Mapping): - problems.extend(_problems(item_loc, f"expected an object, got {item!r}")) - continue - field = cast("Mapping[object, object]", item) - for key in field: - if not isinstance(key, str) or key not in _STRUCT_FIELD_KEYS: - problems.extend(_problems(item_loc, f"unexpected key {key!r}", "unknown_key")) - for key in sorted(_STRUCT_FIELD_KEYS - field.keys()): - problems.extend(_problems((*item_loc, key), "missing required key", "missing_key")) - if "name" in field and not isinstance(field["name"], str): - problems.extend(_problems((*item_loc, "name"), "expected a string")) - if "data_type" in field: - problems.extend(_check_data_type_field(field["data_type"], (*item_loc, "data_type"))) - return tuple(problems) - - -_SCALAR_MAP_KEYS: Final = frozenset(ScalarMap.__annotations__) - - -def _check_scalar_map_entries( - value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple): - return _problems(loc, f"expected an array (tuple) of [old, new] pairs, got {value!r}") - problems: list[ValidationProblem] = [] - for index, item in enumerate(cast("tuple[object, ...]", value)): - if not isinstance(item, tuple) or len(cast("tuple[object, ...]", item)) != 2: - problems.extend(_problems((*loc, index), f"expected an [old, new] pair, got {item!r}")) - continue - for position, scalar in enumerate(cast("tuple[object, ...]", item)): - problems.extend(_check_json_value(scalar, (*loc, index, position))) - return tuple(problems) - - -def _check_scalar_map(value: object, loc: tuple[str | int, ...]) -> tuple[ValidationProblem, ...]: - if not isinstance(value, Mapping): - return _problems(loc, f"expected an object, got {value!r}") - mapping = cast("Mapping[object, object]", value) - problems: list[ValidationProblem] = [] - for key in mapping: - if not isinstance(key, str) or key not in _SCALAR_MAP_KEYS: - problems.extend(_problems((*loc,), f"unexpected key {key!r}", "unknown_key")) - for key in _SCALAR_MAP_KEYS: - if key in mapping: - problems.extend(_check_scalar_map_entries(mapping[key], (*loc, key))) - return tuple(problems) - - -def _check_rectilinear_dim_spec( - value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - if not isinstance(value, bool) and isinstance(value, int): - return () - if not isinstance(value, tuple): - return _problems( - loc, - f"expected an integer or an array of integers / [value, count] pairs, got {value!r}", - ) - problems: list[ValidationProblem] = [] - for index, item in enumerate(cast("tuple[object, ...]", value)): - if not isinstance(item, bool) and isinstance(item, int): - continue - if isinstance(item, tuple) and len(cast("tuple[object, ...]", item)) == 2: - problems.extend( - problem - for position, part in enumerate(cast("tuple[object, ...]", item)) - for problem in _check_json_int(part, (*loc, index, position)) - ) - continue - problems.extend( - _problems((*loc, index), f"expected an integer or a [value, count] pair, got {item!r}") - ) - return tuple(problems) - - -def _check_rectilinear_dim_specs( - value: object, loc: tuple[str | int, ...] -) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple): - return _problems(loc, f"expected an array (tuple) of dimension specs, got {value!r}") - return tuple( - problem - for index, item in enumerate(cast("tuple[object, ...]", value)) - for problem in _check_rectilinear_dim_spec(item, (*loc, index)) - ) - - -@dataclass(frozen=True, slots=True) -class _EntityShape: - """Shape facts for one known entity name, derived from its TypedDicts.""" - - object_keys: frozenset[str] - configuration_required: bool - config_keys: frozenset[str] - config_required: frozenset[str] - config_checkers: Mapping[str, _FieldChecker] - invariants: tuple[_EntityInvariant, ...] = () - - -def _shape( - object_type: type, - configuration_type: type, - checkers: Mapping[str, _FieldChecker], - invariants: tuple[_EntityInvariant, ...] = (), -) -> _EntityShape: - config_keys = frozenset(configuration_type.__annotations__) - if frozenset(checkers) != config_keys: - raise AssertionError( # pragma: no cover - registry construction guard - f"checkers {sorted(checkers)} do not cover configuration keys {sorted(config_keys)}" - ) - return _EntityShape( - object_keys=frozenset(object_type.__annotations__), - configuration_required="configuration" - in cast("frozenset[str]", object_type.__required_keys__), # type: ignore[attr-defined] - config_keys=config_keys, - config_required=cast( - "frozenset[str]", - configuration_type.__required_keys__, # type: ignore[attr-defined] - ), - config_checkers=dict(checkers), - invariants=invariants, - ) - - -def _bare_shape() -> _EntityShape: - """Shape of an entity that takes no configuration. - - Both spellings are valid: the spec makes `{"name": ...}` the base form - and permits the bare short-hand when no configuration is required. The - object form is therefore accepted with an absent or empty - `configuration`, and any member inside one is an unknown key. - - Keyword arguments deliberately: this is a six-field record whose flags - are easy to transpose positionally. - """ - return _EntityShape( - object_keys=frozenset({"name", "configuration", "must_understand"}), - configuration_required=False, - config_keys=frozenset(), - config_required=frozenset(), - config_checkers={}, - ) - - -_CODEC_SHAPES: Final[Mapping[str, _EntityShape]] = { - BLOSC_CODEC_NAME: _shape( - BloscCodecObject, - BloscCodecConfiguration, - { - "cname": _literal(BLOSC_CNAME), - "clevel": _int_in_range(0, 9), - "shuffle": _literal(BLOSC_SHUFFLE), - "blocksize": _check_non_negative_int, - "typesize": _check_positive_int, - }, - invariants=(_blosc_typesize_is_present_when_shuffling,), - ), - BYTES_CODEC_NAME: _shape( - BytesCodecObject, BytesCodecConfiguration, {"endian": _literal(ENDIANNESS)} - ), - CAST_VALUE_CODEC_NAME: _shape( - CastValueCodecObject, - CastValueCodecConfiguration, - { - "data_type": _check_data_type_field, - "rounding": _literal(CAST_ROUNDING_MODE), - "out_of_range": _literal(CAST_OUT_OF_RANGE_MODE), - "scalar_map": _check_scalar_map, - }, - ), - CRC32C_CODEC_NAME: _shape(Crc32cCodecObject, Empty, {}), - GZIP_CODEC_NAME: _shape( - GzipCodecObject, GzipCodecConfiguration, {"level": _int_in_range(0, 9)} - ), - SCALE_OFFSET_CODEC_NAME: _shape( - ScaleOffsetCodecObject, - ScaleOffsetCodecConfiguration, - {"offset": _check_json_value, "scale": _check_json_value}, - ), - SHARDING_INDEXED_CODEC_NAME: _shape( - ShardingIndexedCodecObject, - ShardingIndexedCodecConfiguration, - { - "chunk_shape": _check_int_tuple, - "codecs": _check_field_tuple, - "index_codecs": _check_field_tuple, - "index_location": _literal(SHARDING_INDEX_LOCATION), - }, - ), - TRANSPOSE_CODEC_NAME: _shape( - TransposeCodecObject, TransposeCodecConfiguration, {"order": _check_permutation} - ), - ZSTD_CODEC_NAME: _shape( - ZstdCodecObject, - ZstdCodecConfiguration, - {"level": _int_in_range(-131072, 22), "checksum": _check_json_bool}, - ), -} - -_CHUNK_GRID_SHAPES: Final[Mapping[str, _EntityShape]] = { - REGULAR_CHUNK_GRID_NAME: _shape( - RegularChunkGridObject, RegularChunkGridConfiguration, {"chunk_shape": _check_int_tuple} - ), - RECTILINEAR_CHUNK_GRID_NAME: _shape( - RectilinearChunkGridObject, - RectilinearChunkGridConfiguration, - { - # "inline" is the sole member of the kind Literal; the type - # exports no constant tuple for it. - "kind": _literal(("inline",)), - "chunk_shapes": _check_rectilinear_dim_specs, - }, - ), -} - -_CHUNK_KEY_ENCODING_SHAPES: Final[Mapping[str, _EntityShape]] = { - DEFAULT_CHUNK_KEY_ENCODING_NAME: _shape( - DefaultChunkKeyEncodingObject, - DefaultChunkKeyEncodingConfiguration, - {"separator": _literal(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)}, - ), - V2_CHUNK_KEY_ENCODING_NAME: _shape( - V2ChunkKeyEncodingObject, - V2ChunkKeyEncodingConfiguration, - {"separator": _literal(V2_CHUNK_KEY_ENCODING_SEPARATOR)}, - ), -} - -_BARE_DATA_TYPE_NAMES: Final = ( - BOOL_DATA_TYPE_NAME, - INT8_DATA_TYPE_NAME, - INT16_DATA_TYPE_NAME, - INT32_DATA_TYPE_NAME, - INT64_DATA_TYPE_NAME, - UINT8_DATA_TYPE_NAME, - UINT16_DATA_TYPE_NAME, - UINT32_DATA_TYPE_NAME, - UINT64_DATA_TYPE_NAME, - FLOAT16_DATA_TYPE_NAME, - FLOAT32_DATA_TYPE_NAME, - FLOAT64_DATA_TYPE_NAME, - COMPLEX64_DATA_TYPE_NAME, - COMPLEX128_DATA_TYPE_NAME, - RAW_BYTES_FAMILY, - BYTES_DATA_TYPE_NAME, - STRING_DATA_TYPE_NAME, -) - -_DATA_TYPE_SHAPES: Final[Mapping[str, _EntityShape]] = { - **{name: _bare_shape() for name in _BARE_DATA_TYPE_NAMES}, - NUMPY_DATETIME64_DATA_TYPE_NAME: _shape( - NumpyDatetime64, - NumpyDatetime64Configuration, - {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _int_in_range(1, 2**31 - 1)}, - ), - NUMPY_TIMEDELTA64_DATA_TYPE_NAME: _shape( - NumpyTimedelta64, - NumpyTimedelta64Configuration, - {"unit": _literal(NUMPY_TIME_UNIT), "scale_factor": _int_in_range(1, 2**31 - 1)}, - ), - STRUCT_DATA_TYPE_NAME: _shape(Struct, StructConfiguration, {"fields": _check_struct_fields}), -} - - -def _validate_known_entity( - value: object, name: str, shape: _EntityShape, entity: str -) -> tuple[ValidationProblem, ...]: - """Every reason `value` is not an instance of `name`'s canonical type. - - Locations are relative to the entry itself (`("configuration", key)` - etc.); callers prefix the entry's position in its document. - """ - if isinstance(value, str): - if not shape.configuration_required: - return () - return _problems( - (), - f"{entity} {name!r} requires a configuration and has no bare short-hand " - f"form; use {{'name': {name!r}, 'configuration': {{...}}}}", - "invalid_value", - ) - if not isinstance(value, Mapping): - return _problems((), f"expected a bare name or an object, got {value!r}") - mapping = cast("Mapping[object, object]", value) - problems: list[ValidationProblem] = [] - for key in mapping: - if not isinstance(key, str) or key not in shape.object_keys: - problems.extend(_problems((), f"unexpected key {key!r}", "unknown_key")) - if "must_understand" in mapping: - problems.extend(_check_json_bool(mapping["must_understand"], ("must_understand",))) - if "configuration" not in mapping: - if shape.configuration_required: - problems.extend( - _problems( - ("configuration",), - f"{entity} {name!r} requires a 'configuration' object", - "missing_key", - ) - ) - return tuple(problems) - configuration = mapping["configuration"] - if not isinstance(configuration, Mapping): - problems.extend(_problems(("configuration",), f"expected an object, got {configuration!r}")) - return tuple(problems) - config = cast("Mapping[object, object]", configuration) - for key in config: - if not isinstance(key, str) or key not in shape.config_keys: - problems.extend(_problems(("configuration",), f"unexpected key {key!r}", "unknown_key")) - for key in sorted(shape.config_required - {k for k in config if isinstance(k, str)}): - problems.extend( - _problems( - ("configuration", key), - f"configuration for {entity} {name!r} is missing required key {key!r}", - "missing_key", - ) - ) - for key, checker in shape.config_checkers.items(): - if key in config: - problems.extend(checker(config[key], ("configuration", key))) - if len(problems) == 0: - # Invariants read members directly, so they run only once every - # member has been vouched for; a complaint about one member is - # reason enough not to reason across them. - usable = cast("Mapping[str, object]", config) - for invariant in shape.invariants: - problems.extend( - ValidationProblem(("configuration", *found.loc), found.message, found.kind) - for found in invariant(usable) - ) - return tuple(problems) - - -def validate_known_codec_metadata(value: object) -> tuple[ValidationProblem, ...] | None: - """Shape problems for a known-name codec entry, or None if not judged. - - None means the entry has no interpretable name or its name is not a - codec this package defines (extension openness: unknown entities are - not ours to judge). An empty list means `value` is an instance of the - named codec's canonical metadata type. - """ - name = entity_name(value) - if name is None: - return None - shape = _CODEC_SHAPES.get(name) - if shape is None: - return None - return _validate_known_entity(value, name, shape, "codec") - - -def validate_known_chunk_grid_metadata(value: object) -> tuple[ValidationProblem, ...] | None: - """Shape problems for a known-name chunk grid entry, or None if not judged.""" - name = entity_name(value) - if name is None: - return None - shape = _CHUNK_GRID_SHAPES.get(name) - if shape is None: - return None - return _validate_known_entity(value, name, shape, "chunk grid") - - -_ENTITY_SHAPES: Final[Mapping[ExtensionPointField, Mapping[str, _EntityShape]]] = { - DATA_TYPE: _DATA_TYPE_SHAPES, - CODECS: _CODEC_SHAPES, - CHUNK_GRID: _CHUNK_GRID_SHAPES, - CHUNK_KEY_ENCODING: _CHUNK_KEY_ENCODING_SHAPES, -} - - -def validate_known_entity_metadata( - field: ExtensionPointField, value: object -) -> tuple[ValidationProblem, ...] | None: - """Shape problems for an entity known at `field`, or None if not judged.""" - name = entity_name(value) - if name is None: - return None - shape = _ENTITY_SHAPES.get(field, {}).get(canonical_name(field, name)) - if shape is None: - return None - return _validate_known_entity(value, name, shape, field.replace("_", " ").rstrip("s")) - - -def entity_configuration_keys(field: ExtensionPointField, name: str) -> frozenset[str] | None: - """Every configuration member `name` models at `field`, or None if unmodelled. - - The registry checks a rule's declared `reads` against this, so a rule - cannot claim to read a member that does not exist. - """ - shape = _ENTITY_SHAPES.get(field, {}).get(canonical_name(field, name)) - return None if shape is None else shape.config_keys - - -def entity_required_configuration_keys( - field: ExtensionPointField, name: str -) -> frozenset[str] | None: - """The configuration members `name` requires, or None if unmodelled. - - A rule may subscript only these: a required member that is absent or - ill-typed is reported at `("configuration", member)`, which stands the - rule down. An optional member can be legitimately absent with no - problem reported, so a rule that subscripts one raises `KeyError` out - of a validator instead of returning a verdict. - """ - shape = _ENTITY_SHAPES.get(field, {}).get(canonical_name(field, name)) - return None if shape is None else shape.config_required - - -def modelled_entities() -> frozenset[tuple[ExtensionPointField, str]]: - """Every `(extension point, name)` with a shape validator.""" - return frozenset((field, name) for field, shapes in _ENTITY_SHAPES.items() for name in shapes) - - -def blocking_problems( - problems: Sequence[ValidationProblem], -) -> tuple[ValidationProblem, ...]: - """The problems that prevent interpreting an entity's fields. - - `unknown_key` problems do not: a member this package does not model - says nothing about the members it does. Rules use this so a single - unrecognized key cannot silently suppress every other judgment about - the same entity — the extra key is still reported, and the geometry - checks still run. - """ - return tuple(problem for problem in problems if problem.kind != "unknown_key") - - -__all__ = [ - "blocking_problems", - "entity_configuration_keys", - "entity_name", - "entity_required_configuration_keys", - "modelled_entities", - "validate_known_chunk_grid_metadata", - "validate_known_codec_metadata", - "validate_known_entity_metadata", -] diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index fe91d7ffc8..2aa5bdda5a 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -7,7 +7,8 @@ import pytest from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.rules._chunk_grid import ChunkGrid, shard_index_grid +from zarr_metadata.v3._parts import ChunkGrid, shard_index_grid +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS if TYPE_CHECKING: from collections.abc import Mapping @@ -84,28 +85,31 @@ def _u(*lengths: int) -> tuple[frozenset[int], ...]: } +def _grid_of(grid: object, shape: object) -> ChunkGrid: + """The grid `grid` describes over an array of `shape`. + + A grid entity builds its own; one out of scope pins only the rank the + array shape gives it. + """ + name = grid if isinstance(grid, str) else (grid or {}).get("name") # type: ignore[union-attr] + entity_type = CORE_AND_EXTENSIONS.resolve("chunk_grid", name) if isinstance(name, str) else None + if entity_type is None: + return ChunkGrid.unreadable(shape) + entity, _ = entity_type.coerce(grid, CORE_AND_EXTENSIONS) + if entity is None: + return ChunkGrid.unreadable(shape) + return entity.grid(shape) # type: ignore[attr-defined] + + @pytest.mark.parametrize(("grid", "shape", "rank", "extents"), GRIDS.values(), ids=list(GRIDS)) def test_chunk_grid_of(grid: object, shape: object, rank: object, extents: object) -> None: - built = ChunkGrid.of(grid, shape) + built = _grid_of(grid, shape) assert built.rank == rank assert built.extents == extents -def test_an_unmodelled_grid_is_carried_verbatim() -> None: - # A rule for a third-party grid can still read its own configuration. - grid = {"name": "mycorp.hilbert", "configuration": {"order": 3}} - assert ChunkGrid.of(grid, (64, 64)).metadata == grid - - -def test_a_derived_grid_carries_no_metadata() -> None: - # Nothing may validate a grid this package invented, or report a - # location into one, so it must not look like a document's grid. - assert ChunkGrid.regular((8, 8)).metadata is None - assert ChunkGrid.of(REGULAR, (64, 64)).permuted((1, 0)).metadata is None - - def test_permuting_reorders_the_axes() -> None: - grid = ChunkGrid.of(_rectilinear(((30, 34), (32, 32))), (64, 64)) + grid = _grid_of(_rectilinear(((30, 34), (32, 32))), (64, 64)) assert grid.permuted((1, 0)).extents == (frozenset({32}), frozenset({30, 34})) # An order that is not a permutation of the rank keeps the rank only. assert grid.permuted((0, 1, 2)).extents is None diff --git a/packages/zarr-metadata/tests/rules/test_registry.py b/packages/zarr-metadata/tests/rules/test_registry.py deleted file mode 100644 index f333a550ec..0000000000 --- a/packages/zarr-metadata/tests/rules/test_registry.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Tests for rule registration. - -The registry exists so that a rule cannot be defined without being run. -These tests cover the three ways that could still fail: a rule declaring -dependencies no such document has, an entity whose rules were never -imported, and a document rule set assembled from something other than -the registry. -""" - -from __future__ import annotations - -import pkgutil -from typing import TYPE_CHECKING - -import pytest - -import zarr_metadata.rules._entities as entities -from zarr_metadata.rules import ( - ZARR_V2_ARRAY_RULES, - ZARR_V3_ARRAY_RULES, - ZARR_V3_GROUP_RULES, - Rule, -) -from zarr_metadata.rules._registry import ( - dispatched_fields, - document_rule, - entity_rule, - register_document_type, - registered_entities, -) -from zarr_metadata.rules._v3_array import ZARR_V3_ARRAY -from zarr_metadata.v3._extension_points import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, - RAW_BYTES_FAMILY, -) -from zarr_metadata.v3._shape import modelled_entities - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.model._validation import ValidationProblem - from zarr_metadata.rules._spec import ArrayParts -from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME -from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME - -# Entities the package models that carry no *composition* rule — nothing -# about them depends on the document or on the codec chain. Several still -# have value constraints (`blosc`'s clevel range, `gzip`'s and `zstd`'s -# level ranges); those are refinements of the type and live with it in -# `v3._shape`, not here. Listed by hand, keyed by extension point, so that -# adding a codec is a deliberate choice between "write a rule" and "record -# that composition says nothing", never a silent omission. -_RULE_FREE = frozenset( - { - (CODECS, "blosc"), - (CODECS, "crc32c"), - (CODECS, "gzip"), - (CODECS, "scale_offset"), - (CODECS, "zstd"), - (CHUNK_KEY_ENCODING, "default"), - (CHUNK_KEY_ENCODING, "v2"), - (DATA_TYPE, "bool"), - (DATA_TYPE, "int8"), - (DATA_TYPE, "int16"), - (DATA_TYPE, "int32"), - (DATA_TYPE, "int64"), - (DATA_TYPE, "uint8"), - (DATA_TYPE, "uint16"), - (DATA_TYPE, "uint32"), - (DATA_TYPE, "uint64"), - (DATA_TYPE, "float16"), - (DATA_TYPE, "float32"), - (DATA_TYPE, "float64"), - (DATA_TYPE, "complex64"), - (DATA_TYPE, "complex128"), - (DATA_TYPE, RAW_BYTES_FAMILY), - (DATA_TYPE, "bytes"), - (DATA_TYPE, "string"), - } -) - - -def test_every_shape_modelled_entity_is_accounted_for() -> None: - # Every shape-modelled entity either - # carries rules or is recorded as deliberately rule-free. - assert modelled_entities() == registered_entities() | _RULE_FREE - - -def test_every_shape_modelled_field_has_a_dispatcher() -> None: - # Regression: shapes existed for four extension points but dispatchers - # for only two, so `entity_rule` accepted registrations at `data_type` - # and `chunk_key_encoding` whose rules then silently never ran — the - # exact silent-pass failure the registry exists to prevent. A rule can - # only fire at a field something dispatches. - shape_modelled = {field for field, _ in modelled_entities()} - assert shape_modelled <= dispatched_fields() - - -def test_every_field_with_rules_has_a_dispatcher() -> None: - assert {field for field, _ in registered_entities()} <= dispatched_fields() - - -def test_rule_free_entities_really_have_no_rules() -> None: - # Guards the exclusion list itself: an entity cannot be listed as - # rule-free while quietly carrying rules. - assert registered_entities() & _RULE_FREE == frozenset() - - -def test_rules_are_keyed_by_extension_point_not_name() -> None: - # `bytes` is a core codec and a registered extension data type; a rule - # for one must never fire on the other, so the key carries the field. - assert {(CODECS, "bytes"), (DATA_TYPE, "bytes")} <= modelled_entities() - assert (CODECS, "bytes") in registered_entities() - assert (DATA_TYPE, "bytes") not in registered_entities() - - -def test_every_entity_module_is_imported() -> None: - # The package auto-imports its modules; this asserts the discovery - # actually ran, so a new module cannot sit unimported and inert. - module_names = {info.name for info in pkgutil.iter_modules(entities.__path__)} - assert len(module_names) != 0 - for name in module_names: - assert f"{entities.__name__}.{name}" in __import__("sys").modules - - -@pytest.mark.parametrize("rules", [ZARR_V3_ARRAY_RULES, ZARR_V2_ARRAY_RULES, ZARR_V3_GROUP_RULES]) -def test_rule_sets_are_non_empty(rules: tuple[object, ...]) -> None: - assert len(rules) != 0 - - -def test_error_document_rule_requiring_an_unknown_key() -> None: - # A rule whose dependency is misspelled can never fire, and a rule - # that never fires is indistinguishable from one that always passes. - with pytest.raises(ValueError, match="could never fire"): - - @document_rule(ZARR_V3_ARRAY, frozenset({"shapee"})) - def _misspelled(document: object) -> tuple[()]: # pragma: no cover - never runs - return () - - -def test_error_entity_rule_requiring_an_unknown_key() -> None: - with pytest.raises(ValueError, match="could never fire"): - - @entity_rule(ZARR_V3_ARRAY, CHUNK_GRID, "regular", requires=frozenset({"shapee"})) - def _misspelled( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, - ) -> tuple[()]: # pragma: no cover - refused at registration - return () - - -def test_error_entity_rule_for_an_unmodelled_entity() -> None: - # Entity rules read configuration members by name, so a rule for an - # entity with no shape validator could never fire. - with pytest.raises(ValueError, match="no shape validator"): - - @entity_rule(ZARR_V3_ARRAY, CHUNK_GRID, "hilbert") - def _unmodelled( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, - ) -> tuple[()]: # pragma: no cover - refused at registration - return () - - -def test_error_entity_rule_for_name_modelled_only_at_another_extension_point() -> None: - # `regular` has a chunk-grid shape, but no codec shape. Name-only lookup - # would accept this registration and later interpret codec metadata using - # the chunk-grid schema. - with pytest.raises(ValueError, match="no shape validator"): - - @entity_rule(ZARR_V3_ARRAY, CODECS, "regular") - def _wrong_extension_point( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, - ) -> tuple[()]: # pragma: no cover - refused at registration - return () - - -def test_error_rule_for_an_unregistered_document_type() -> None: - with pytest.raises(LookupError, match="unknown document type"): - - @document_rule("zarr_v9_array", frozenset()) - def _orphan(document: object) -> tuple[()]: # pragma: no cover - never runs - return () - - -def test_register_document_type_accepts_declared_extension_keys() -> None: - register_document_type("test_doc", frozenset({"a"}), extension_keys=frozenset({"b"})) - - @document_rule("test_doc", frozenset({"a", "b"})) - def _uses_both(document: object) -> tuple[()]: - return () - - assert _uses_both.requires == frozenset({"a", "b"}) - - -def test_error_entity_rule_reads_an_unmodelled_member() -> None: - with pytest.raises(ValueError, match="does not model"): - - @entity_rule(ZARR_V3_ARRAY, CODECS, GZIP_CODEC_NAME, reads=frozenset({"nosuchmember"})) - def _unmodelled_member( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, - ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered - return () - - -def test_error_entity_rule_reads_an_optional_member() -> None: - # Only a required member is safe to subscript: an absent optional one is - # reported by nothing, so the rule would raise out of a validator. - with pytest.raises(ValueError, match="reads_optional"): - - @entity_rule(ZARR_V3_ARRAY, CODECS, BYTES_CODEC_NAME, reads=frozenset({"endian"})) - def _subscripts_an_optional_member( - configuration: Mapping[str, object], - document: Mapping[str, object], - incoming: ArrayParts | None, - ) -> tuple[ValidationProblem, ...]: # pragma: no cover - never registered - return () - - -# Every document rule, by the layer it belongs to. A field rule reads one -# top-level field; a composition rule spans several. Listed by hand so that -# adding one is a deliberate choice, the way `_RULE_FREE` makes "this entity -# has no composition rule" a deliberate choice. -_FIELD_RULES = frozenset( - { - "_check_data_type_spelling", - "_check_data_type_shape", - "_check_chunk_key_encoding_shape", - "_check_chunk_grid_shape", - "check_codec_pipeline_order", - "check_codec_shapes", - "check_chunk_grid_shape", - "_dispatch_chunk_grid_entity_rules", - "_dispatch_data_type_entity_rules", - "_dispatch_chunk_key_encoding_entity_rules", - "_dispatch_codecs_entity_rules", - "check_consolidated_entries", - } -) -_COMPOSITION_RULES = frozenset( - {"_check_fill_matches_dtype", "check_dimension_names_length", "check_chunks_match_shape"} -) - - -@pytest.mark.parametrize( - "rules", - [ZARR_V3_ARRAY_RULES, ZARR_V2_ARRAY_RULES, ZARR_V3_GROUP_RULES], - ids=["v3-array", "v2-array", "v3-group"], -) -def test_document_rules_are_classified_by_what_they_read(rules: tuple[Rule, ...]) -> None: - # The classification is not decoration: a rule reading one field is a - # value constraint on that field, and could in principle move down to - # the layer that owns the field. One spanning fields cannot. - for rule in rules: - name = rule.check.__name__ - assert name in _FIELD_RULES | _COMPOSITION_RULES, f"{name} is classified nowhere" - if name in _FIELD_RULES: - assert len(rule.requires) == 1, ( - f"{name} is a field rule but reads {sorted(rule.requires)}" - ) - else: - assert len(rule.requires) >= 2, f"{name} is a composition rule but reads one field" diff --git a/packages/zarr-metadata/tests/rules/test_spec_propagation.py b/packages/zarr-metadata/tests/rules/test_spec_propagation.py deleted file mode 100644 index 26456a016b..0000000000 --- a/packages/zarr-metadata/tests/rules/test_spec_propagation.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Tests for propagating an array's parts through a codec chain. - -The property under test: every codec is judged against the array it -*receives*, which is the document's chunk only for the first codec in -the chain. Anything that transforms the array — a transpose, a cast, a -shard — changes what the next codec sees. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.rules._chunk_grid import ChunkGrid -from zarr_metadata.rules._spec import ArrayParts, propagate, transitions_registered -from zarr_metadata.v3.codec.kind import ARRAY_ARRAY_CODEC_NAMES - -if TYPE_CHECKING: - from collections.abc import Mapping - - -def _doc(codecs: tuple[object, ...], chunk: tuple[int, ...] = (6, 4)) -> Mapping[str, object]: - return { - "zarr_format": 3, - "node_type": "array", - "shape": (12, 8), - "data_type": "uint8", - "fill_value": 0, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": chunk}}, - "chunk_key_encoding": "default", - "codecs": codecs, - } - - -def _transpose(*order: int) -> Mapping[str, object]: - return {"name": "transpose", "configuration": {"order": order}} - - -def _shard( - inner: tuple[int, ...], - codecs: tuple[object, ...] = ("bytes",), - index_codecs: tuple[object, ...] = ({"name": "bytes", "configuration": {"endian": "little"}},), -) -> Mapping[str, object]: - return { - "name": "sharding_indexed", - "configuration": {"chunk_shape": inner, "codecs": codecs, "index_codecs": index_codecs}, - } - - -# (document, expected verdict). The chunk is (6, 4); a transpose (1, 0) in -# front of a shard means the shard receives (4, 6), and the verdict must -# follow the transposed shape, not the grid. -CASES: dict[str, tuple[Mapping[str, object], bool]] = { - "shard-alone-divides": (_doc((_shard((3, 2)),)), True), - "shard-alone-does-not-divide": (_doc((_shard((4, 3)),)), False), - # Regression: before propagation these two verdicts were reversed. - "transpose-then-shard-divides-transposed": (_doc((_transpose(1, 0), _shard((2, 3)))), True), - "transpose-then-shard-does-not-divide-transposed": ( - _doc((_transpose(1, 0), _shard((3, 2)))), - False, - ), - "two-transposes-cancel": (_doc((_transpose(1, 0), _transpose(1, 0), _shard((3, 2)))), True), - # Inside a shard the incoming array is the inner chunk, so a nested - # transpose is judged against the inner chunk's rank, and a nested - # shard against the transposed inner chunk. - "nested-transpose-matches-inner-rank": ( - _doc((_shard((3, 2), codecs=(_transpose(1, 0), "bytes")),)), - True, - ), - "nested-shard-follows-nested-transpose": ( - # inner chunk (3, 2) transposed -> (2, 3); nested shard (2, 1) divides it. - _doc((_shard((3, 2), codecs=(_transpose(1, 0), _shard((2, 1)))),)), - True, - ), - "nested-shard-violates-transposed-inner": ( - # inner chunk (3, 2) transposed -> (2, 3); nested shard (3, 1): 3 does not divide 2. - _doc((_shard((3, 2), codecs=(_transpose(1, 0), _shard((3, 1)))),)), - False, - ), - # An unknown codec might change the shape, so downstream geometry - # declines rather than guessing — an otherwise-invalid shard passes. - "unknown-codec-stops-propagation": (_doc(({"name": "zfpy"}, _shard((4, 3)))), True), - # The shard index is a uint64 array, so a bytes codec inside - # `index_codecs` needs an endianness like any multi-byte encoding. - "index-codecs-bare-bytes-needs-endian": ( - _doc((_shard((3, 2), index_codecs=("bytes", "crc32c")),)), - False, - ), -} - - -@pytest.mark.parametrize(("doc", "valid"), CASES.values(), ids=list(CASES)) -def test_verdict_follows_the_incoming_array(doc: Mapping[str, object], valid: bool) -> None: - problems = validate_array_metadata_v3(doc) - assert (len(problems) == 0) is valid, [str(p) for p in problems] - - -def test_error_locates_the_offending_shard() -> None: - problems = validate_array_metadata_v3(_doc((_transpose(1, 0), _shard((3, 2))))) - assert [(p.loc, p.kind) for p in problems] == [ - (("codecs", 1, "configuration", "chunk_shape", 0), "invalid_value") - ] - - -def test_propagate_yields_incoming_spec_per_codec() -> None: - from zarr_metadata.rules._registry import entity_configuration - from zarr_metadata.v3._extension_points import CODECS - - chain = (_transpose(1, 0), "bytes", "crc32c") - start = ArrayParts(ChunkGrid.regular((6, 4)), "uint8") - seen = list(propagate(chain, start, lambda c: entity_configuration(CODECS, c))) - incoming = [spec for _, _, spec in seen] - assert incoming[0] == ArrayParts(ChunkGrid.regular((6, 4)), "uint8") - # bytes receives the transposed chunk - assert incoming[1] == ArrayParts(ChunkGrid.regular((4, 6)), "uint8") - # past the array->bytes boundary there is no array to describe - assert incoming[2] is None - - -def test_cast_value_changes_the_downstream_data_type() -> None: - from zarr_metadata.rules._registry import entity_configuration - from zarr_metadata.v3._extension_points import CODECS - - chain = ({"name": "cast_value", "configuration": {"data_type": "float32"}}, "bytes") - start = ArrayParts(ChunkGrid.regular((6, 4)), "uint8") - seen = list(propagate(chain, start, lambda c: entity_configuration(CODECS, c))) - assert seen[1][2] == ArrayParts(ChunkGrid.regular((6, 4)), "float32") - - -def test_unknown_codec_yields_nothing_known() -> None: - from zarr_metadata.rules._registry import entity_configuration - from zarr_metadata.v3._extension_points import CODECS - - start = ArrayParts(ChunkGrid.regular((6, 4)), "uint8") - seen = list( - propagate(({"name": "zfpy"}, "bytes"), start, lambda c: entity_configuration(CODECS, c)) - ) - assert seen[1][2] is None - - -def test_every_array_array_codec_registers_a_transition() -> None: - # A modelled array->array codec with no transition is treated as - # unknown and stops propagation, standing down every rule downstream - # of it. No exemptions: a codec that changes nothing registers the - # identity and says so. - assert set(ARRAY_ARRAY_CODEC_NAMES) <= transitions_registered() - - -def test_error_transition_for_a_non_array_array_codec() -> None: - from zarr_metadata.rules._spec import spec_transition - - with pytest.raises(ValueError, match="only array->array codecs"): - - @spec_transition("gzip") - def _nope(configuration: object, incoming: ArrayParts) -> ArrayParts: # pragma: no cover - return incoming diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 9be104a0c5..5a8482eba8 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -443,7 +443,9 @@ def test_error_consolidated_nested_group_recursion() -> None: def test_error_group_parse_raises() -> None: from zarr_metadata.rules import parse_group_metadata_v3 - with pytest.raises(MetadataValidationError, match="fill_value invalid"): + with pytest.raises( + MetadataValidationError, match=r"consolidated_metadata\.metadata\.a\.fill_value" + ): parse_group_metadata_v3( { "zarr_format": 3, diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 55766080da..5858cc375b 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -302,8 +302,6 @@ def test_all_is_grouped_and_unique() -> None: "NumpyTimedelta64", "ProblemKind", "RectilinearDimSpec", - "Rule", - "RuleCheck", "ScalarMap", "ScalarMapEntry", "ShardingIndexLocation", diff --git a/packages/zarr-metadata/tests/test_registry_drift.py b/packages/zarr-metadata/tests/test_registry_drift.py deleted file mode 100644 index b678c17e62..0000000000 --- a/packages/zarr-metadata/tests/test_registry_drift.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Drift tests tying the hand-written judgment registries to the package's -type modules: adding a codec, chunk grid, or data type module without -registering it in the corresponding judgment surface must fail a test -rather than silently weaken validation (an unregistered codec, for -example, would suppress the exactly-one-array->bytes check for every -pipeline containing it).""" - -from __future__ import annotations - -import importlib -import pkgutil - -import zarr_metadata.v3.chunk_grid -import zarr_metadata.v3.chunk_key_encoding -import zarr_metadata.v3.codec -import zarr_metadata.v3.data_type -from zarr_metadata.rules._storage_class import storage_class -from zarr_metadata.rules._v3_array import ( - _check_fill_for_dtype, # pyright: ignore[reportPrivateUsage] -) -from zarr_metadata.v3._extension_points import RAW_BYTES_FAMILY -from zarr_metadata.v3._shape import ( # pyright: ignore[reportPrivateUsage] - _CHUNK_GRID_SHAPES, - _CHUNK_KEY_ENCODING_SHAPES, - _CODEC_SHAPES, - _DATA_TYPE_SHAPES, -) -from zarr_metadata.v3.codec.kind import codec_kind_of_name - - -def _module_constants(package: object, suffix: str) -> set[str]: - """Values of `*` constants across a package's public modules.""" - names: set[str] = set() - for info in pkgutil.iter_modules(package.__path__): # type: ignore[attr-defined] - if info.name.startswith("_"): - continue - module = importlib.import_module(f"{package.__name__}.{info.name}") # type: ignore[attr-defined] - names.update( - value - for attribute, value in vars(module).items() - if attribute.endswith(suffix) and isinstance(value, str) - ) - return names - - -def test_every_codec_module_is_kind_classified() -> None: - codec_names = _module_constants(zarr_metadata.v3.codec, "_CODEC_NAME") - assert codec_names, "constant scan found nothing — the naming convention moved?" - unclassified = {name for name in codec_names if codec_kind_of_name(name) is None} - assert not unclassified - - -def test_every_codec_module_has_a_shape_validator() -> None: - codec_names = _module_constants(zarr_metadata.v3.codec, "_CODEC_NAME") - assert codec_names == set(_CODEC_SHAPES) - - -def test_every_chunk_grid_module_has_a_shape_validator() -> None: - grid_names = _module_constants(zarr_metadata.v3.chunk_grid, "_CHUNK_GRID_NAME") - assert grid_names, "constant scan found nothing — the naming convention moved?" - assert grid_names == set(_CHUNK_GRID_SHAPES) - - -def test_every_chunk_key_encoding_module_has_a_shape_validator() -> None: - names = _module_constants(zarr_metadata.v3.chunk_key_encoding, "_CHUNK_KEY_ENCODING_NAME") - assert names, "constant scan found nothing — the naming convention moved?" - assert names == set(_CHUNK_KEY_ENCODING_SHAPES) - - -def test_every_data_type_module_has_a_shape_validator() -> None: - names = _module_constants(zarr_metadata.v3.data_type, "_DATA_TYPE_NAME") - assert names, "constant scan found nothing — the naming convention moved?" - assert names | {RAW_BYTES_FAMILY} == set(_DATA_TYPE_SHAPES) - - -def test_every_data_type_has_a_fill_value_branch() -> None: - # object() is a valid fill value for no data type this package - # defines, so a known name must produce a complaint; only genuinely - # unknown names may decline (extension openness). The parameterized - # r family has no name constant and is represented by "r8". - dtype_names = _module_constants(zarr_metadata.v3.data_type, "_DATA_TYPE_NAME") - assert dtype_names, "constant scan found nothing — the naming convention moved?" - unjudged = { - name for name in {*dtype_names, "r8"} if _check_fill_for_dtype(name, object()) is None - } - assert not unjudged - - -def test_every_data_type_module_has_a_storage_class() -> None: - # The bytes codec's endianness rule and the struct field rule both ask - # this question, so an unclassified data type silently disables both. - # `struct` classifies from its fields, so it is sampled with one; the - # r family has no name constant and is represented by "r8". - dtype_names = _module_constants(zarr_metadata.v3.data_type, "_DATA_TYPE_NAME") - assert dtype_names, "constant scan found nothing — the naming convention moved?" - samples: dict[str, object] = { - "struct": { - "name": "struct", - "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}, - } - } - unclassified = { - name for name in {*dtype_names, "r8"} if storage_class(samples.get(name, name)) is None - } - assert not unclassified From 27a0bd9a63a5f9dea4db5372e30e709615c8bf2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 17:33:20 +0200 Subject: [PATCH 035/107] docs(zarr-metadata): changelog describes what ships, not the route there Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.bugfix.md | 19 ++-- .../zarr-metadata/changes/4379.feature.6.md | 5 +- .../zarr-metadata/changes/4379.feature.7.md | 48 +++++----- .../zarr-metadata/changes/4379.feature.md | 91 ++++++++++--------- packages/zarr-metadata/changes/4379.misc.1.md | 17 ++++ packages/zarr-metadata/changes/4379.misc.md | 21 +++-- 6 files changed, 114 insertions(+), 87 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.misc.1.md diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md index 42dbed80a6..756cc6baf4 100644 --- a/packages/zarr-metadata/changes/4379.bugfix.md +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -13,10 +13,11 @@ rules layer: or inner chunk shape went unreported. Every chunk of an array has the array's rank whatever the grid, so the rank is kept and only the geometry checks stand down. -- One unusable configuration member suppressed every other rule about - the same entity — a misspelled `index_location` hid the problems in a - shard's inner pipelines. Rules now declare the members they read and - stand down individually, so the rest of the entity is still judged. +- One unusable configuration member suppressed every other judgment + about the same entity — a misspelled `index_location` hid the problems + in a shard's inner pipelines. An optional member that fails its type + check now falls back to absent, so the rest of the entity is still + judged. - Problems about an entity as a whole pointed at a `configuration` node that a bare-string entity does not have (`("codecs", 0, "configuration")` for `"codecs": ["bytes"]`). They now point at the @@ -24,13 +25,13 @@ rules layer: - The endianness message named no data type, so inside a shard's `index_codecs` it appeared to be about the array's own type rather than the shard index's `uint64`. It names the type. -- A malformed `must_understand` — part of an entity's envelope, not of its - configuration — counted as "the configuration is unreadable" and stood - down every composition rule for that entity. Only a problem at the - entity itself or at `configuration` as a whole does that now. +- A malformed `must_understand` — part of an entity's envelope, not of + its configuration — counted as "the configuration is unreadable" and + stood down every judgment for that entity. It no longer does: it is + not a configuration member, so it cannot make one unreadable. The three derivations of "what array does this pipeline encode?" are now -one abstraction, `rules._chunk_grid`, which answers per dimension rather +one abstraction, `v3._parts`, which answers per dimension rather than per grid, and plurally: what a codec is handed is `ArrayParts`, every part of an array the pipeline will encode, carrying the chunk grid itself rather than a summary of one. Every chain rule is a statement about all of diff --git a/packages/zarr-metadata/changes/4379.feature.6.md b/packages/zarr-metadata/changes/4379.feature.6.md index e39cb20165..e55a5e8476 100644 --- a/packages/zarr-metadata/changes/4379.feature.6.md +++ b/packages/zarr-metadata/changes/4379.feature.6.md @@ -1,13 +1,14 @@ Unknown members inside a *known* entity's `configuration` (e.g. an extra key in a `blosc` configuration) now report as their own `unknown_key` -problem kind, and no longer suppress the other rules about that entity. +problem kind, and no longer suppress the other judgments about that +entity. Whether configurations are closed remains unspecified ([zarr-specs#270](https://github.com/zarr-developers/zarr-specs/issues/270)), and this package retains its strict reading deliberately: in practice an unmodelled member is a typo or a setting meant for a different entity, and accepting it silently means silently ignoring what the writer asked for. -Judging it is the rules layer's job: `rules.parse_*` and the +Judging it is the semantic layer's job: `rules.parse_*` and the whole-document pydantic field types reject it, while `model.parse_*` and the bare `ZarrV3MetadataField` type — neither of which interprets entity configurations — accept it. The dedicated kind exists so that a consumer diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 3c845ea767..f33f0e1e02 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -1,30 +1,28 @@ -Value constraints live with the type they refine, not in the rule layer. -The package now distinguishes three judgments rather than two: **type** -(is this an integer?), **value** (is it an integer in `[0, 9]`?), and -**composition** (does this codec's rank match the array that reached it?). -Only the third needs a document or a codec chain, and only the third is a -rule. +Three judgments rather than two: **type** (is this an integer?), +**value** (is it an integer in `[0, 9]`?), and **composition** (does this +codec's rank match the array that reached it?). Each one is answered by +the entity it is about, because all three are facts about that entity. So `blosc`'s `clevel` range, `gzip`'s and `zstd`'s `level` ranges, `typesize` being positive, `blocksize` being non-negative, a `transpose` -`order` being a permutation of its own indices, and a time data type's -`scale_factor` range are all stated where the member is typed, in -`v3._shape`. `blosc`'s "typesize is required unless shuffle is -`noshuffle`" spans two members of one configuration, so it is an entity -invariant there. Seven of the 23 entity rules became declarations, three -rule modules are gone, and what is left in `rules._entities` is the eight -judgments that genuinely read the document or the chain. +`order` being a permutation of its own indices, a time data type's +`scale_factor` range, and `blosc`'s "typesize is required unless shuffle +is `noshuffle`" are all stated on the class that has those members. So +are the composition answers, each taking the part of the document it +needs. -The boundary is the *member*, for a reason worth recording: a verdict from -the shape layer marks a whole member unusable, so a constraint on the -elements *within* one — every extent of a `chunk_shape` being positive — -would cost precision elsewhere. A zero on one axis would stop a -rectilinear grid reporting the axis beside it, and would stand down a -shard's inner pipeline entirely. Those stay in the rule layer, which -judges element by element. +Three name-keyed tables disappear with them, replaced by a method on the +class that knew the answer all along: which storage class a data type +has (and how a `struct` folds its fields), what a codec does to the +array it receives, and what a chunk grid divides an array into. -Document-level rules follow the same layout: `_v3_array` is split into -field rules (one top-level field, such as codec pipeline ordering) and -composition rules (spanning fields, such as fill value against data type), -and a test asserts the partition so that adding a rule is a deliberate -choice about which it is. +What a fill value may be is the data type's answer too, and the families +are where the widths live: every integer type differs from every other +only in its bounds, every float only in its hex parser, every complex +only in its component type. + +The boundary is worth recording: an *optional* member that fails its +type check falls back to absent, because a bad `index_location` says +nothing about whether a shard's pipelines are well formed and silencing +them would lose a real judgment. A *required* one stops the entity, +because there is no honest reading of a `blosc` whose level is a string. diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index e1ce8faf95..bb04fa9a61 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -1,53 +1,62 @@ -Added `zarr_metadata.rules`: composition rules for full metadata -documents. The package now validates metadata in two layers with one -contract each — `model` checks structure element by element and `rules` -judges composition across the document. Rules are registered where they -are defined; rules about a particular codec, chunk grid, or data type -live with that entity under `rules._entities` and are dispatched by name, -so adding an entity adds a module there and changes nothing else. +Added `zarr_metadata.rules`: semantic validation for full metadata +documents. The package now validates in two layers with one contract +each — `model` checks structure element by element, and `rules` judges +what structure alone cannot: whether the values are in range and whether +the document's parts agree with one another. -- **Rule sets**: `ZARR_V3_ARRAY_RULES` covers fill value vs. data type, - codec pipeline kind ordering, known-name shapes, - dimension-name counts, chunk-grid values (positive extents) and - geometry (regular rank; rectilinear rank and per-dimension chunk-size - sums, RLE pairs included), transpose orders (self-permutation at any - depth, rank agreement with `shape`), and sharding (inner `codecs` and - `index_codecs` judged as pipelines recursively at every nesting depth; - inner chunk shapes positive, rank-matched, and evenly dividing the - enclosing chunk, recursively). New `ZARR_V2_ARRAY_RULES` - (chunks/shape rank agreement) and `ZARR_V3_GROUP_RULES` (inline - consolidated metadata recurses, judging each embedded child document - by its own rules at its path). +The second layer is not a rule engine. Each extension — every codec, +data type, chunk grid and chunk key encoding — is a class that answers +for itself: + +- `coerce` reads raw metadata into the entity, or says why it is not one; +- `problems` says which of its own values the spec disallows; +- `to_json` gives its simplest equivalent spelling; +- `incoming_problems`, `shape_problems`, `fill_value_problems` and + `transition` answer the questions that need a piece of the document: + the array reaching a codec, the shape a grid divides, the fill value a + data type must accept, what the next codec sees. + +Validating a document is composing those answers. Nothing in the +document layer knows what `blosc` or `int32` or `rectilinear` is, so +adding an extension is a class and one registry entry. + +- **What is judged**: fill value against data type (recursively, through + `struct` fields and `cast_value` targets), codec pipeline kind + ordering, dimension-name counts, chunk-grid extents and geometry + (regular rank; rectilinear rank and per-dimension chunk-size sums, RLE + pairs included), transpose orders (self-permutation, and rank + agreement with the array that reaches them at any depth), and sharding + (inner `codecs` and `index_codecs` judged as pipelines recursively; + inner chunk shapes positive, rank-matched, and evenly dividing every + chunk they will be handed). v2 arrays get chunks/shape rank agreement; + v3 groups recurse into inline consolidated metadata, judging each + embedded child document at its path. +- **Scope**: `v3._registry` maps each extension point to the + identifiers in play, in two scopes — `CORE`, what the specification + defines, and `CORE_AND_EXTENSIONS`, which adds what `zarr-extensions` + registers and this package models. A name in neither is not rejected; + it is simply not judged. The scope is a `Context`, passed to every + `coerce` and ignored by every entity that does not contain another. - **Read-side entry points**: `validate_*` / `parse_*` for array and group documents in both format versions mirror the model layer's - grammar with a stronger judgment — structure *and* composition, every + grammar with a stronger judgment — structure *and* semantics, every problem reported together, JSON arrays normalized to tuples before - judgment. There is no `is_*` counterpart: a composition-invalid + judgment. There is no `is_*` counterpart: a semantically invalid document is still an instance of the TypedDict, so only the structural layer can narrow honestly — use `zarr_metadata.model.is_*` for that. -- **Boundary change**: two composition checks that lived in the - structural validator moved here — v3 `dimension_names` vs `shape` and - v2 `chunks` vs `shape` rank agreement. `zarr_metadata.model`'s - validators, parsers, and dataclasses now accept those documents (they - are lossless, structurally well-formed representations of what a store - may contain); use the `rules` validators to judge them. This also removes - the double report the overlap used to produce. +- **Boundary change**: two checks that lived in the structural validator + moved here — v3 `dimension_names` vs `shape` and v2 `chunks` vs + `shape` rank agreement. `zarr_metadata.model`'s validators, parsers + and dataclasses now accept those documents (they are lossless, + structurally well-formed representations of what a store may contain); + use the `rules` validators to judge them. This also removes the double + report the overlap used to produce. - **Strictness stance**, now documented on the package: `zarr_metadata` models canonical documents and is deliberately stricter than any given implementation; implementations coerce ambiguous input as they see fit and then validate the canonical result. - -- **Codec chains are judged against the array each codec receives**: - `transpose` permutes the shape and `cast_value` changes the data type - seen by everything after it, so a shard behind a transpose must divide - the transposed chunk, and a `bytes` codec behind a cast needs an - endianness for the *target* type. `zarr_metadata.v3.codec.kind` sorts - known codec names into the spec's three pipeline kinds. -- **Nested data types** in struct fields and cast targets recursively run - their registered composition rules, including time scale factors and - nested struct field constraints. -- **Pydantic field types** for array and group documents now run the - composition rules as well as structural validation. +- **Pydantic field types** for array and group documents run the + semantic checks as well as structural validation. Known follow-up: v2 fill-value/dtype consistency (NumPy dtype grammar) -has no rule yet. +has no check yet. diff --git a/packages/zarr-metadata/changes/4379.misc.1.md b/packages/zarr-metadata/changes/4379.misc.1.md new file mode 100644 index 0000000000..3cdfba2401 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.misc.1.md @@ -0,0 +1,17 @@ +The rule registry is gone, and 3900 lines with it. It existed to +dispatch a rule to the entity it was about, gate it on whether that +entity's metadata was readable, and propagate what each codec did to the +array — all three of which the entities now do themselves. + +Deleted: the registry and its `reads`/`reads_optional` gate, the rule +engine, the nine modules of rules keyed by entity name, the fill-value +dispatch table, three frozensets of data type names with a hand-rolled +recursion for `struct`, four tables of per-member type checks, and the +drift tests that existed to keep those tables in step with the +TypedDicts. The tables cannot drift now, because there is one of each +fact instead of four. + +What replaces the drift tests is a correspondence test: an entity's +dataclass fields, its configuration TypedDict, and its member table are +three spellings of one set, and whether the bare-name spelling is +allowed follows from the TypedDict's required keys. diff --git a/packages/zarr-metadata/changes/4379.misc.md b/packages/zarr-metadata/changes/4379.misc.md index 429e163c3c..f6acce53dd 100644 --- a/packages/zarr-metadata/changes/4379.misc.md +++ b/packages/zarr-metadata/changes/4379.misc.md @@ -4,11 +4,11 @@ the recursive `JSONValue` alias and then resolves `ReadOnly`, `closed`, `NotRequired` and `Literal` unaided. The pre-existing totality test feeds arbitrary JSON to the document -validators; a random object never names a codec, so it dispatched no -entity rule at all. Drawing codecs from their own types and assembling the -chain by pipeline kind (`array->array`* `array->bytes` `bytes->bytes`*) -dispatches an entity rule for every document, where arbitrary JSON -dispatched none; 86% report a composition problem, against 78% for a flat +validators; a random object never names a codec, so no codec was ever +asked about itself. Drawing codecs from their own types and assembling +the chain by pipeline kind (`array->array`* `array->bytes` +`bytes->bytes`*) reaches a real codec in every document, where arbitrary +JSON reached none; 86% report a problem, against 78% for a flat list of the same codecs. Ordering earns that by guaranteeing an `array->bytes` codec is present, not by avoiding an early exit — nothing short-circuits on a misordered chain. @@ -25,11 +25,12 @@ that silently stops dispatching fails loudly instead of leaving a green property run that tests nothing. `st.from_type` honours the TypedDicts exactly, so it cannot produce a -configuration member of the wrong *type* — which is precisely what the -per-member `reads` gate exists to stand rules down for. A corrupting -strategy covers that: four separate one-token changes to the gate make -`validate_*` raise `TypeError` on ordinary malformed metadata, and none of -them was detectable before. +configuration member of the wrong *type* — which is exactly the case +where an entity has to decide what it can still say about itself. A +corrupting strategy covers that, and it is what established that the +entity-based validator returns the same verdicts as the rule registry it +replaced: identical problems on valid documents, identical verdicts on +corrupted ones. The package's tests are now type-checked along with its sources. They were checked by nothing: pyright was configured for `src` alone, and the repo's From ca4368c62964aa12a6294a5028e1218a7cb4fde4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:05:54 +0200 Subject: [PATCH 036/107] fix(zarr-metadata): judge a nested metadata field like a top-level one Found by adversarial review. An extra member, a `configuration` that is not an object, and a `must_understand` that is not a boolean are all rejected at the top level and were all *accepted* one level in -- inside a shard's pipelines, a struct field's data type, a cast_value target. The canonicalizer then deleted them, so an invalid document came back valid and smaller. The old shape layer checked the envelope wherever it ran, including inside a shard; `named_configuration` silently tolerates all three. The fix is to reuse the model layer's own `validate_metadata_field_v3` in `Context.coerce`, because a metadata field is a metadata field wherever it appears. The document's own fields pass `envelope_judged=True`, the structural layer having already said so. The differential over 2267 documents missed this: `st.from_type` honours the TypedDicts, so it cannot emit a malformed envelope. Two more from the same review. `ShardingIndexedCodec.problems()` was the one site composing a location by hand instead of with `within`, so a codec inside a shard reported at a path that does not exist in the document. And `CORE` and `CORE_AND_EXTENSIONS` shared one dict for `chunk_key_encoding` -- registering there would have reached both, and the subset test was vacuous for that field. A test now walks every reported location into the document. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_document.py | 4 +- .../src/zarr_metadata/v3/_registry.py | 38 +++++++-- .../v3/codec/sharding_indexed.py | 6 +- .../zarr-metadata/tests/v3/test_entities.py | 85 +++++++++++++++++++ 4 files changed, 119 insertions(+), 14 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index f7aa370fb9..03b8ad9a82 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -95,14 +95,14 @@ def read_array_v3( if value is None: read[key] = None continue - entity, found = context.coerce(field, value, (key,)) + entity, found = context.coerce(field, value, (key,), envelope_judged=True) read[key] = entity problems.extend(found) codecs: list[MetadataEntity | object] = [] entries = document.get("codecs") if isinstance(entries, (list, tuple)): for index, entry in enumerate(cast("Sequence[object]", entries)): - codec, found = context.coerce(CODECS, entry, ("codecs", index)) + codec, found = context.coerce(CODECS, entry, ("codecs", index), envelope_judged=True) codecs.append(codec) problems.extend(found) return ( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 7a51307a8c..48610c5f71 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -23,7 +23,10 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Final -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import ( + ValidationProblem, + validate_metadata_field_v3, +) from zarr_metadata.v3._entity import named_configuration from zarr_metadata.v3._extension_points import ( CHUNK_GRID, @@ -109,7 +112,12 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] return entity def coerce( - self, field: ExtensionPointField, value: object, loc: Loc = () + self, + field: ExtensionPointField, + value: object, + loc: Loc = (), + *, + envelope_judged: bool = False, ) -> tuple[MetadataEntity | object, tuple[ValidationProblem, ...]]: """One nested entity, read in this scope. @@ -121,20 +129,34 @@ def coerce( `loc` prefixes the problems, so they point at where in the containing configuration the entity sat. + + A metadata field is a metadata field wherever it appears, so the + envelope gets the same structural judgment here that the model + layer gives a top-level one -- an extra member, a `configuration` + that is not an object, a `must_understand` that is not a boolean. + `envelope_judged` says that judgment has already happened, which + it has for the fields of a document the model layer accepted. """ + problems: list[ValidationProblem] = [] + if not envelope_judged: + problems.extend( + ValidationProblem((*loc, *found.loc), found.message, found.kind) + for found in validate_metadata_field_v3(value) + ) name, _, _ = named_configuration(value) if name is None: return value, ( + *problems, ValidationProblem(loc, f"expected a metadata field, got {value!r}", "invalid_type"), ) entity_type = self.resolve(field, name) if entity_type is None: - return value, () - entity, problems = entity_type.coerce(value, self) - located = tuple( - ValidationProblem((*loc, *found.loc), found.message, found.kind) for found in problems + return value, tuple(problems) + entity, found = entity_type.coerce(value, self) + problems.extend( + ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found ) - return (value if entity is None else entity), located + return (value if entity is None else entity), tuple(problems) _CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { @@ -204,7 +226,7 @@ def coerce( CODECS: {**_CORE_CODECS, **_EXTENSION_CODECS}, DATA_TYPE: {**_CORE_DATA_TYPES, **_EXTENSION_DATA_TYPES}, CHUNK_GRID: {**_CORE_CHUNK_GRIDS, **_EXTENSION_CHUNK_GRIDS}, - CHUNK_KEY_ENCODING: _CORE_CHUNK_KEY_ENCODINGS, + CHUNK_KEY_ENCODING: {**_CORE_CHUNK_KEY_ENCODINGS}, } ) """What the specification defines, plus what `zarr-extensions` registers.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 0141ce0166..289556e2db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -22,6 +22,7 @@ one_of, problem, sequence_of, + within, ) from zarr_metadata.v3._parts import ( UNKNOWN_GRID, @@ -190,10 +191,7 @@ def problems(self) -> tuple[ValidationProblem, ...]: for member in ("codecs", "index_codecs"): for position, codec in enumerate(cast("tuple[object, ...]", getattr(self, member))): if isinstance(codec, MetadataEntity): - found.extend( - ValidationProblem((member, position, *entry.loc), entry.message, entry.kind) - for entry in codec.problems() - ) + found.extend(within((member, position), codec.problems())) return tuple(found) def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index ae7b04a774..e09bad28ca 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -14,6 +14,7 @@ import pytest +from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, @@ -212,3 +213,87 @@ def test_an_unknown_key_is_reported_without_losing_the_member() -> None: assert [problem.kind for problem in problems] == ["unknown_key"] assert codec is not None assert codec.scalar_map == {"encode": (), "enc": ()} + + +# (a defect in a nested metadata field, the location it belongs at) +NESTED_ENVELOPES: dict[str, tuple[dict[str, object], tuple[str | int, ...]]] = { + "unexpected-member": ({"name": "bytes", "typo": 1}, ("typo",)), + "configuration-not-an-object": ({"name": "bytes", "configuration": 42}, ("configuration",)), + "must-understand-not-a-boolean": ( + {"name": "bytes", "must_understand": "no"}, + ("must_understand",), + ), +} + + +@pytest.mark.parametrize( + ("codec", "inner_loc"), NESTED_ENVELOPES.values(), ids=list(NESTED_ENVELOPES) +) +def test_a_nested_metadata_field_is_judged_like_a_top_level_one( + codec: dict[str, object], inner_loc: tuple[str | int, ...] +) -> None: + # A metadata field is a metadata field wherever it appears. These sit + # inside a shard's pipeline, which only the entity layer reads, so + # nothing else is going to notice them. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (8,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (8,)}}, + "chunk_key_encoding": "default", + "codecs": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4,), + "codecs": (codec,), + "index_codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + }, + }, + ), + } + problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + assert [problem.loc for problem in problems] == [ + ("codecs", 0, "configuration", "codecs", 0, *inner_loc) + ] + + +def test_every_problem_location_indexes_into_the_document() -> None: + # Two defects in one shard, at different depths. A location that does + # not resolve is a location a consumer cannot use. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (8, 8), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (8, 8)}}, + "chunk_key_encoding": "default", + "codecs": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4, 4), + "codecs": ( + {"name": "transpose", "configuration": {"order": (0, 0)}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ), + "index_codecs": ({"name": "bytes"},), + }, + }, + ), + } + problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + assert len(problems) == 2 + for problem in problems: + node: object = document + for step in problem.loc: + if not isinstance(node, (dict, tuple)) or (isinstance(node, dict) and step not in node): + # A `missing_key` problem names where the key belongs, so + # it is allowed to run past the end of what is there. Any + # other kind must address a node that exists. + assert problem.kind == "missing_key", (problem.loc, step) + break + node = node[step] # type: ignore[index] From 4fcb5d7721edffa50e781074673d685539c6c9af Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:13:39 +0200 Subject: [PATCH 037/107] fix(zarr-metadata): absence is UNSET, and a bad member hides less Four more from the adversarial review. `None` meant absent in the entity layer, which contradicts the package's own invariant that `None` is a JSON `null` the document wrote. It cost a real distinction: `scale_offset` with `{"offset": null}` canonicalized to the bare name, silently erasing what was written. Optional members now default to `UNSET`, as the model layer's do, and `null` survives -- and is reported, because no data type admits it as a scalar. A stray configuration key on an `r` data type returned no entity, so its fill value went unjudged and the document looked valid to anyone taking the documented tolerant reading (collect problems, filter `unknown_key`). An unknown key is survivable here as everywhere else. `blosc` rejected a `typesize` below 1 even under `noshuffle`, where the spec says the value is ignored and `configuration` deletes it -- so `to_json` turned an invalid codec into a valid document. `canonicalize_array_metadata_v3` was not idempotent on `json.loads` output: the per-field simplifications test for `tuple`, and the validator was normalizing on a copy the canonicalizer never saw. And a member that cannot be read no longer hides the values of the ones beside it. The entity is built from what did read, its value problems are reported, and anything the defaults would say about an unreadable member is dropped. Measured against the rule registry over one shared corpus of 800 documents: no verdict differs in the laxer direction, the 400 valid documents are identical, and documents losing a report fall from 98 to 34 -- the residue being composition judgments, which need an entity that could not be built. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/rules/_canonical.py | 10 ++- .../src/zarr_metadata/rules/_documents.py | 9 ++- .../src/zarr_metadata/v3/_entity.py | 54 ++++++++++++---- .../v3/chunk_key_encoding/default.py | 3 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 3 +- .../src/zarr_metadata/v3/codec/blosc.py | 11 +++- .../src/zarr_metadata/v3/codec/bytes.py | 5 +- .../src/zarr_metadata/v3/codec/cast_value.py | 7 ++- .../zarr_metadata/v3/codec/scale_offset.py | 22 ++++++- .../v3/codec/sharding_indexed.py | 3 +- .../src/zarr_metadata/v3/codec/zstd.py | 3 +- .../src/zarr_metadata/v3/data_type/raw.py | 9 ++- .../zarr-metadata/tests/v3/test_entities.py | 61 +++++++++++++++++++ 13 files changed, 170 insertions(+), 30 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py index eb449b97af..5d0412e15a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py @@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast from zarr_metadata.model._array import ZarrV3ArrayMetadata +from zarr_metadata.model._validation import arrays_to_tuples from zarr_metadata.rules._documents import validate_array_metadata_v3 from zarr_metadata.v3._document import read_array_v3 from zarr_metadata.v3._entity import MetadataEntity @@ -93,10 +94,15 @@ def canonicalize_array_metadata_v3( same way -- but the structural problems come back too, and the result is `Invalid` rather than a canonical document. """ - problems = validate_array_metadata_v3(document) + normalized = cast("ZarrV3ArrayMetadataJSON", arrays_to_tuples(document)) + problems = validate_array_metadata_v3(normalized) if len(problems) != 0: return Invalid(problems) - canonical = _canonical_document(document) + # Normalized first, so a document spelled with JSON arrays reaches the + # same fixpoint as the tuple spelling. It did not: the per-field + # simplifications test for `tuple`, and the validator was normalizing + # on a copy the canonicalizer never saw. + canonical = _canonical_document(normalized) # The model layer's round trip normalizes the fields no entity owns. return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index 3c43ce0250..151e59a1e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -76,7 +76,14 @@ def _judged( def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: - """Every reason `value` is not a valid v3 array document. + """Why `value` is not a valid v3 array document. + + Every structural problem, and every semantic problem that can be + determined. One member that cannot be read costs the *composition* + judgments about the entity holding it -- whether a shard's inner + shape divides the array it is handed cannot be answered by a shard + that could not be built -- so a document with two defects in one + configuration may need a second pass. The verdict is never affected. Structural problems (from the model layer) and semantic problems (from the entities themselves) are reported together. JSON arrays are diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 40b30db2a9..d9f4feecb1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -44,6 +44,7 @@ from typing_extensions import TypeIs +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem, is_json from zarr_metadata.v3._parts import ChunkGrid @@ -195,12 +196,12 @@ def _as_tuples(value: object) -> object: def coerce_members( configuration: Mapping[str, object], types: MemberTypes -) -> tuple[dict[str, object], tuple[ValidationProblem, ...], bool]: +) -> tuple[dict[str, object], tuple[ValidationProblem, ...], frozenset[str]]: """The members `types` declares, taken from `configuration`. - Returns what was accepted, every problem found, and whether the entity - is still worth building. Three kinds of problem, and they differ in - that last part: + Returns what was accepted, every problem found, and the names of the + required members that could not be read. Three kinds of problem, and + they differ in that last part: - a key the entity does not declare says the value carries something extra, not that it is wrong; @@ -213,7 +214,7 @@ def coerce_members( """ problems: list[ValidationProblem] = [] members: dict[str, object] = {} - usable = True + unreadable: set[str] = set() for key in configuration: if key not in types: problems.extend(problem(("configuration",), f"unexpected key {key!r}", "unknown_key")) @@ -223,7 +224,7 @@ def coerce_members( problems.extend( problem(("configuration", key), f"missing required key {key!r}", "missing_key") ) - usable = False + unreadable.add(key) continue # Normalized before the check, so a check only ever sees the tuples # the TypedDicts declare -- never the lists raw JSON arrives as. @@ -236,8 +237,8 @@ def coerce_members( if all(entry.kind == "unknown_key" for entry in found): members[key] = value elif required: - usable = False - return members, tuple(problems), usable + unreadable.add(key) + return members, tuple(problems), frozenset(unreadable) # No `slots=True`, deliberately: it rebuilds the class, which leaves the @@ -254,6 +255,13 @@ class MetadataEntity: typed `| None` with a default of `None`, so absence is representable and a canonical spelling can leave it out. + Frozen, so an entity of hashable members is hashable. One holding a + value out of scope is not, because that value is the JSON the document + wrote and a JSON object is a `dict` -- the same way any frozen + dataclass holding a list is unhashable. It cannot be an immutable + mapping instead: `MappingProxyType` is unhashable too, and anything + else stops `json.dumps` from serializing what `to_json` returns. + Most subclasses declare `member_types` and nothing else: the default `coerce` and `to_json` are written once here against that table. The ones that override are the ones with something particular to say -- @@ -317,8 +325,28 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: "missing_key", ) configuration = cast("Mapping[str, object]", {}) - members, found, usable = coerce_members(configuration, cls.member_types) - if not usable: + members, found, unreadable = coerce_members(configuration, cls.member_types) + if len(unreadable) != 0: + # The entity cannot be built, but the members that *did* read + # can still be judged -- one bad member should not hide the + # value problems of the ones beside it. Anything the partial + # reading says about an unreadable member is its default + # talking, so those are dropped. + partial = cls(must_understand=must_understand, **members) # type: ignore[arg-type] + found = ( + *found, + # `within`, because a partial reading reports relative to + # the configuration and `coerce`'s caller does not insert + # that segment -- `coerce_members` problems already carry it. + *within( + (), + [ + entry + for entry in partial.problems() + if entry.loc[:1] not in {(key,) for key in unreadable} + ], + ), + ) return None, found return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] @@ -328,11 +356,15 @@ def configuration(self) -> dict[str, object]: Absent optional members are left out, which is what makes the bare-name spelling reachable. Override to drop a member that another member renders meaningless. + + Absence is `UNSET`, never `None`: this package holds `None` to + mean a JSON `null` the document actually wrote, and `scale_offset` + is a real case where `null` and absent are different documents. """ return { key: value for key in type(self).member_types - if (value := getattr(self, key)) is not None + if (value := getattr(self, key)) is not UNSET } def problems(self) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 793482aa8e..20ff86dbbd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -12,6 +12,7 @@ from typing_extensions import TypedDict +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( MemberTypes, MetadataEntity, @@ -75,7 +76,7 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): class DefaultChunkKeyEncoding(MetadataEntity): """The `default` chunk key encoding, coerced from its metadata.""" - separator: DefaultChunkKeyEncodingSeparator | None = None + separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 7bcae4613e..fb417d1ad8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -18,6 +18,7 @@ from typing_extensions import TypedDict +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( MemberTypes, MetadataEntity, @@ -81,7 +82,7 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): class V2ChunkKeyEncoding(MetadataEntity): """The `v2` chunk key encoding, coerced from its metadata.""" - separator: V2ChunkKeyEncodingSeparator | None = None + separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 7dd736e6e9..10367ade91 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -10,6 +10,7 @@ from typing_extensions import TypedDict, Unpack +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, @@ -120,7 +121,7 @@ class BloscCodec(CodecEntity): clevel: int = 5 shuffle: BloscShuffle = "noshuffle" blocksize: int = 0 - typesize: int | None = None + typesize: int | UNSET = UNSET identifier: ClassVar[str] = BLOSC_CODEC_NAME variable_size: ClassVar[bool] = True @@ -157,7 +158,11 @@ def problems(self) -> tuple[ValidationProblem, ...]: "invalid_value", ) ) - if self.typesize is not None and self.typesize < 1: + # Only where it means something: under `noshuffle` the spec says + # "the value is ignored" and `configuration` drops it, so judging + # it would let `to_json` turn an invalid codec into a valid + # document. + if self.typesize is not UNSET and self.shuffle != BLOSC_NO_SHUFFLE and self.typesize < 1: found.extend( problem( ("typesize",), @@ -165,7 +170,7 @@ def problems(self) -> tuple[ValidationProblem, ...]: "invalid_value", ) ) - if self.shuffle != BLOSC_NO_SHUFFLE and self.typesize is None: + if self.shuffle != BLOSC_NO_SHUFFLE and self.typesize is UNSET: found.extend( problem( ("typesize",), diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 69418c9209..983b7a0dd5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -9,6 +9,7 @@ from typing_extensions import TypedDict +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, @@ -88,7 +89,7 @@ class BytesCodec(CodecEntity): has no byte order to state, and the spec lets such an array omit it. """ - endian: Endianness | None = None + endian: Endianness | UNSET = UNSET identifier: ClassVar[str] = BYTES_CODEC_NAME kind: ClassVar[CodecKind] = "array_bytes" @@ -115,7 +116,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb f"bytes codec is not compatible with variable-length data_type {name!r}", "invalid_value", ) - if storage == "multi_byte" and self.endian is None: + if storage == "multi_byte" and self.endian is UNSET: return problem( ("endian",), f"endian is required for data type {name!r}, which contains multi-byte values", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 4e973b2188..ed2a99368a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( DATA_TYPE, @@ -184,9 +185,9 @@ class CastValueCodec(CodecEntity): """ data_type: MetadataEntity | object = None - rounding: CastRoundingMode | None = None - out_of_range: CastOutOfRangeMode | None = None - scalar_map: ScalarMap | None = None + rounding: CastRoundingMode | UNSET = UNSET + out_of_range: CastOutOfRangeMode | UNSET = UNSET + scalar_map: ScalarMap | UNSET = UNSET identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME kind: ClassVar[CodecKind] = "array_array" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index f2d4b9b4ab..c5f7087ee0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -10,11 +10,14 @@ from typing_extensions import TypedDict from zarr_metadata._common import JSONValue +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, MemberTypes, is_json_value, + problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -81,8 +84,8 @@ class ScaleOffsetCodec(CodecEntity): is a question for the rules layer. """ - offset: JSONValue | None = None - scale: JSONValue | None = None + offset: JSONValue | UNSET = UNSET + scale: JSONValue | UNSET = UNSET identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME kind: ClassVar[CodecKind] = "array_array" @@ -100,5 +103,20 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: """ return incoming + def problems(self) -> tuple[ValidationProblem, ...]: + """Each value is a scalar of the array's type, so neither is null. + + The registry says each is "JSON-encoded per the input array's + fill-value rules", and no data type admits `null` as a fill value. + Which scalar it should be needs the data type, so that part is the + document's question, not this codec's. + """ + return tuple( + found + for member in ("offset", "scale") + if getattr(self, member) is None + for found in problem((member,), "expected a scalar, got null", "invalid_value") + ) + def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 289556e2db..1aabeb4701 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._entity import ( @@ -144,7 +145,7 @@ class ShardingIndexedCodec(CodecEntity): chunk_shape: tuple[int, ...] = () codecs: tuple[MetadataEntity | object, ...] = () index_codecs: tuple[MetadataEntity | object, ...] = () - index_location: ShardingIndexLocation | None = None + index_location: ShardingIndexLocation | UNSET = UNSET identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME variable_size: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 65097922ba..772633b037 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -11,6 +11,7 @@ from typing_extensions import TypedDict +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, @@ -81,7 +82,7 @@ class ZstdCodec(CodecEntity): """The `zstd` codec, coerced from its metadata.""" level: int = 0 - checksum: bool | None = None + checksum: bool | UNSET = UNSET identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 6085ded208..797f7a67e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -114,9 +114,14 @@ def coerce(cls, value: object, context: object) -> Coerced[Self]: name, configuration, must_understand = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), "expected an 'r' raw-bytes data type") + found: tuple[ValidationProblem, ...] = () if configuration is not None and len(configuration) != 0: - return None, problem(("configuration",), "'r' takes no configuration", "unknown_key") - return cls(must_understand=must_understand, data_type_name=name), () + # Survivable, as an unknown key is everywhere else: the name + # still says everything this type is, so it is still read and + # its fill values are still judged. Returning nothing here let + # a stray key hide every other problem in the document. + found = problem(("configuration",), "'r' takes no configuration", "unknown_key") + return cls(must_understand=must_understand, data_type_name=name), found def problems(self) -> tuple[ValidationProblem, ...]: """N must be a positive multiple of 8. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index e09bad28ca..c3588f33e6 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -297,3 +297,64 @@ def test_every_problem_location_indexes_into_the_document() -> None: assert problem.kind == "missing_key", (problem.loc, step) break node = node[step] # type: ignore[index] + + +def test_one_unreadable_member_does_not_hide_the_values_of_the_others() -> None: + # `clevel` is the wrong type, so this blosc cannot be built -- but + # `blocksize` was read, and what is wrong with it is still worth + # saying. Losing it would make fixing the document a two-pass job. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (4,)}}, + "chunk_key_encoding": "default", + "codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + { + "name": "blosc", + "configuration": { + "cname": "lz4", + "clevel": "five", + "shuffle": "noshuffle", + "blocksize": -1, + }, + }, + ), + } + problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + assert {problem.loc for problem in problems} == { + ("codecs", 1, "configuration", "clevel"), + ("codecs", 1, "configuration", "blocksize"), + } + + +def test_an_unreadable_member_is_not_judged_by_its_default() -> None: + # `shuffle` could not be read, so it falls back to `noshuffle`, under + # which `typesize` means nothing. The absent `typesize` must not be + # reported as required -- that would be the default talking. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (4,)}}, + "chunk_key_encoding": "default", + "codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + { + "name": "blosc", + "configuration": { + "cname": "lz4", + "clevel": 5, + "shuffle": 7, + "blocksize": 0, + }, + }, + ), + } + problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "shuffle")] From 4cf0455145b776e53d2e65ead29dc9b44d9576f5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:16:46 +0200 Subject: [PATCH 038/107] fix(zarr-metadata): the hygiene the review turned up - `member_types`'s base default was one shared mutable dict; every configuration-free entity had the same object. Now a read-only mapping. - An `unknown_key` problem named the key only in prose, so two of them on one configuration were indistinguishable by location. The location now names it, as the model layer's already did. - Float fill values were unbounded while integers were range-checked, so `float32` accepted `1e39` and stored an infinity. Each width states the largest magnitude it holds; float64 states none, because a Python float is one. - `storage_transformers` was a registerable extension point the document layer never resolved. Nothing models one yet, so nothing is judged there today -- but a registered one is now reached. - The `r` grammar used `\d`, which is every Unicode decimal, so `r` was read as sixteen bits and a third-party name spelled that way was folded into the family. ASCII only; such a name is now left unjudged, as any unmodelled name is. - `RECTILINEAR_CHUNK_GRID_KIND` is exported, as its siblings are. - Three docstrings that had stopped being true: `Coerced` claimed an entity and problems never come back together (a survivable problem does exactly that), `incoming_problems` misstated where its locations are relative to, and the no-slots comment overstated the breakage and did not say CPython fixed it in 3.13. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_document.py | 37 +++++++++++++------ .../src/zarr_metadata/v3/_entity.py | 34 ++++++++++++----- .../v3/chunk_grid/rectilinear.py | 1 + .../zarr_metadata/v3/data_type/_families.py | 14 +++++++ .../src/zarr_metadata/v3/data_type/float16.py | 1 + .../src/zarr_metadata/v3/data_type/float32.py | 1 + .../src/zarr_metadata/v3/data_type/float64.py | 1 + .../src/zarr_metadata/v3/data_type/raw.py | 6 ++- .../tests/rules/test_v3_array_rules.py | 10 +++-- .../tests/v3/test_extension_points.py | 4 +- 10 files changed, 83 insertions(+), 26 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 03b8ad9a82..a93333809c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -26,6 +26,7 @@ CHUNK_KEY_ENCODING, CODECS, DATA_TYPE, + STORAGE_TRANSFORMERS, ChunkGridEntity, DataTypeEntity, ExtensionPointField, @@ -53,6 +54,7 @@ class ArrayDocumentV3: chunk_grid: MetadataEntity | object chunk_key_encoding: MetadataEntity | object codecs: tuple[MetadataEntity | object, ...] + storage_transformers: tuple[MetadataEntity | object, ...] @property def parts(self) -> ArrayParts: @@ -79,6 +81,14 @@ def parts(self) -> ArrayParts: (CHUNK_KEY_ENCODING, "chunk_key_encoding"), ) +# The two the document names as a list. Nothing models a storage +# transformer yet, so nothing is judged there today -- but the extension +# point is registerable, and a registered one has to be reached. +_SEQUENCE_FIELDS: Final[tuple[tuple[ExtensionPointField, str], ...]] = ( + (CODECS, "codecs"), + (STORAGE_TRANSFORMERS, "storage_transformers"), +) + def read_array_v3( document: Mapping[str, object], context: Context @@ -98,20 +108,24 @@ def read_array_v3( entity, found = context.coerce(field, value, (key,), envelope_judged=True) read[key] = entity problems.extend(found) - codecs: list[MetadataEntity | object] = [] - entries = document.get("codecs") - if isinstance(entries, (list, tuple)): - for index, entry in enumerate(cast("Sequence[object]", entries)): - codec, found = context.coerce(CODECS, entry, ("codecs", index), envelope_judged=True) - codecs.append(codec) - problems.extend(found) + sequences: dict[str, tuple[MetadataEntity | object, ...]] = {} + for field, key in _SEQUENCE_FIELDS: + read_entries: list[MetadataEntity | object] = [] + entries = document.get(key) + if isinstance(entries, (list, tuple)): + for index, entry in enumerate(cast("Sequence[object]", entries)): + entity, found = context.coerce(field, entry, (key, index), envelope_judged=True) + read_entries.append(entity) + problems.extend(found) + sequences[key] = tuple(read_entries) return ( ArrayDocumentV3( document=document, data_type=read["data_type"], chunk_grid=read["chunk_grid"], chunk_key_encoding=read["chunk_key_encoding"], - codecs=tuple(codecs), + codecs=sequences["codecs"], + storage_transformers=sequences["storage_transformers"], ), tuple(problems), ) @@ -124,9 +138,10 @@ def _entity_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: entity = getattr(array, key) if isinstance(entity, MetadataEntity): found.extend(within((key,), entity.problems())) - for index, codec in enumerate(array.codecs): - if isinstance(codec, MetadataEntity): - found.extend(within(("codecs", index), codec.problems())) + for _, key in _SEQUENCE_FIELDS: + for index, entity in enumerate(cast("tuple[object, ...]", getattr(array, key))): + if isinstance(entity, MetadataEntity): + found.extend(within((key, index), entity.problems())) return tuple(found) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index d9f4feecb1..04d4ffc5dc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -40,6 +40,7 @@ from collections.abc import Mapping as _Mapping from dataclasses import dataclass, field +from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast from typing_extensions import TypeIs @@ -63,10 +64,15 @@ # `Coerced[Self]` in a return annotation, and not all of them defer # annotation evaluation. Coerced: TypeAlias = tuple[EntityT | None, tuple[ValidationProblem, ...]] -"""The entity, or None and every reason the metadata is not one. +"""The entity if it could be built, and every problem found. -A caller that only wants a verdict reads the problems; one that wants to -go on reading the entity checks for None. Both never happen at once. +One direction holds: no entity means at least one problem. The converse +does not -- a survivable problem (an unknown key, an optional member of +the wrong type) comes back *with* the entity, because the entity is +still readable and saying so is more useful than refusing. + +So test `entity is None` to decide whether to go on reading, and test the +problems to decide the verdict. They are different questions. """ Loc: TypeAlias = "tuple[str | int, ...]" @@ -86,6 +92,7 @@ CHUNK_GRID: Final[ExtensionPointField] = "chunk_grid" CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" CODECS: Final[ExtensionPointField] = "codecs" +STORAGE_TRANSFORMERS: Final[ExtensionPointField] = "storage_transformers" StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How one scalar of a data type occupies bytes. @@ -217,7 +224,9 @@ def coerce_members( unreadable: set[str] = set() for key in configuration: if key not in types: - problems.extend(problem(("configuration",), f"unexpected key {key!r}", "unknown_key")) + problems.extend( + problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") + ) for key, (required, check) in types.items(): if key not in configuration: if required: @@ -241,10 +250,13 @@ def coerce_members( return members, tuple(problems), frozenset(unreadable) -# No `slots=True`, deliberately: it rebuilds the class, which leaves the -# zero-argument `super()` in a subclass pointing at the class that was -# replaced. Subclasses call `super()` to narrow `to_json` and to adjust -# `configuration`, so slots would be a trap laid for every entity. +# No `slots=True`, deliberately. It rebuilds the class, which on Python +# 3.11 and 3.12 leaves the zero-argument `super()` *in that same class's +# body* pointing at the class it replaced. Several entities call `super()` +# to narrow `to_json` and to adjust `configuration`, so they would each +# have to spell it `super(Cls, self)`. CPython fixed this in 3.13, so when +# that is the floor this is worth revisiting; the memory saved is small at +# document scale, which is why it has not been. @dataclass(frozen=True) class MetadataEntity: """One named entity, coerced from its metadata. @@ -283,7 +295,7 @@ class MetadataEntity: an invented identifier that no real name can collide with. """ - member_types: ClassVar[MemberTypes] = {} + member_types: ClassVar[MemberTypes] = MappingProxyType({}) """The configuration members, and the type each one takes. The same keys as the configuration TypedDict, which is the same as the @@ -414,7 +426,8 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb `incoming` is None once the chain can no longer say what reaches here, and the default answer to that is nothing: declining beats - guessing. Locations are relative to this codec's entry. + guessing. Locations are relative to this codec's `configuration`, + as `problems`' are; an empty one lands on the codec itself. """ return () @@ -536,6 +549,7 @@ def named_configuration( "CHUNK_KEY_ENCODING", "CODECS", "DATA_TYPE", + "STORAGE_TRANSFORMERS", "ChunkGridEntity", "CodecEntity", "CodecKind", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 2333a47a59..596dc69739 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -111,6 +111,7 @@ def canonical_chunk_shapes( __all__ = [ + "RECTILINEAR_CHUNK_GRID_KIND", "RECTILINEAR_CHUNK_GRID_NAME", "RectilinearChunkGrid", "RectilinearChunkGridConfiguration", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index fdf04a5357..5364dfacfd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -86,8 +86,22 @@ class FloatDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "multi_byte" hex_parser: ClassVar[Callable[[str], object]] + largest: ClassVar[float | None] + """The largest finite magnitude this width holds, or None for float64. + + None because a Python float *is* a float64, so no literal that reaches + here can exceed it. + """ + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: if is_integer(value) or isinstance(value, float): + largest = type(self).largest + if largest is not None and abs(value) > largest: + return problem( + loc, + f"expected a {type(self).identifier} value, got {value!r}", + "invalid_value", + ) return () if not isinstance(value, str): return problem(loc, f"expected a number or string, got {value!r}", "invalid_value") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 42121410b1..d6504868d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -86,4 +86,5 @@ class Float16DataType(FloatDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float16) + largest: ClassVar[float | None] = 65504.0 identifier: ClassVar[str] = FLOAT16_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 4c589a052a..5e2b287d52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -86,4 +86,5 @@ class Float32DataType(FloatDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float32) + largest: ClassVar[float | None] = 3.4028235e38 identifier: ClassVar[str] = FLOAT32_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 3b24fc999c..b52b9205bd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -87,4 +87,5 @@ class Float64DataType(FloatDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float64) + largest: ClassVar[float | None] = None identifier: ClassVar[str] = FLOAT64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 797f7a67e8..05607462e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -38,9 +38,13 @@ unforgeable by a real name. """ -RAW_BYTES_NAME_PATTERN: Final = re.compile(r"^r(\d+)$") +RAW_BYTES_NAME_PATTERN: Final = re.compile(r"^r([0-9]+)$") """The *shape* of a raw-bytes data type name, not its validity. +ASCII digits only: `\\d` would also match every other Unicode decimal, so +`r\uff11\uff16` would be read as sixteen bits and a genuine third-party +name spelled that way would be folded into this family. + Matches every `r` spelling including malformed ones (`r0`, `r12`), so that a misspelled member of this family is recognized as belonging to it and reported as a misspelling, rather than passing as an unknown diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 5a8482eba8..94c2fe43ea 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -474,7 +474,11 @@ def test_unknown_configuration_member_has_its_own_kind() -> None: "codecs": ({"name": "bytes", "configuration": {"endian": "little", "hint": 1}},), } problems = validate_array_metadata_v3(doc) - assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "unknown_key")] + # The location names the offending key, so a consumer can route the + # problem without parsing the message. + assert [(p.loc, p.kind) for p in problems] == [ + (("codecs", 0, "configuration", "hint"), "unknown_key") + ] def test_error_known_data_type_has_invalid_configuration() -> None: @@ -511,7 +515,7 @@ def test_unknown_member_does_not_mask_a_codec_rule() -> None: "codecs": ({"name": "transpose", "configuration": {"order": (5, 5), "hint": 1}}, "bytes"), } kinds = {(p.loc, p.kind) for p in validate_array_metadata_v3(doc)} - assert (("codecs", 0, "configuration"), "unknown_key") in kinds + assert (("codecs", 0, "configuration", "hint"), "unknown_key") in kinds assert (("codecs", 0, "configuration", "order"), "invalid_value") in kinds @@ -521,7 +525,7 @@ def test_unknown_member_does_not_mask_a_chunk_grid_rule() -> None: "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,), "hint": 1}}, } kinds = {(p.loc, p.kind) for p in validate_array_metadata_v3(doc)} - assert (("chunk_grid", "configuration"), "unknown_key") in kinds + assert (("chunk_grid", "configuration", "hint"), "unknown_key") in kinds assert (("chunk_grid", "configuration", "chunk_shape"), "invalid_value") in kinds diff --git a/packages/zarr-metadata/tests/v3/test_extension_points.py b/packages/zarr-metadata/tests/v3/test_extension_points.py index 7a539fe28d..edbfea6560 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_points.py +++ b/packages/zarr-metadata/tests/v3/test_extension_points.py @@ -54,7 +54,9 @@ def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: "codecs": ({"name": "bytes", "configuration": {"width": 7}},), } problems = validate_array_metadata_v3(document) - assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "unknown_key")] + assert [(p.loc, p.kind) for p in problems] == [ + (("codecs", 0, "configuration", "width"), "unknown_key") + ] def test_forging_the_family_sentinel_cannot_change_a_verdict() -> None: From 0870f8971d202b19b998a32de98dbde812d068a9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:22:13 +0200 Subject: [PATCH 039/107] feat(zarr-metadata): a public door to the extension layer Three independent reviews reached the same conclusion from different directions: the entity layer answers the questions a reader has and the questions an extension author has, and none of it was reachable. Thirty of the thirty-four names an extension needs were private, and the validators took no scope, so the only way to have a third-party entity judged was to mutate a module-level dict at import time. `zarr_metadata.v3.entity` re-exports the layer: the four base classes, `Context` and the two scopes, `ArrayParts`/`ChunkGrid`, the member checks an entity's `member_types` is built from, and the data-type families. No code moved; this is the door, not a new room. `validate_array_metadata_v3`, `parse_array_metadata_v3`, `validate_group_metadata_v3`, `parse_group_metadata_v3` and `canonicalize_array_metadata_v3` take `context=`, defaulting to what this package models. A group carries it down into its consolidated children. Two guards turn the silent failures the review found into errors where the mistake is: - a subclass that does not declare `identifier` (or `kind`, or `scalar_storage`) type-checks cleanly and then raises `AttributeError` from whichever method runs first. `__init_subclass__` says so at import. `base=True` opts out the classes that exist to add a class variable rather than to be an entity. - an optional member with a default other than `UNSET` is emitted for every instance, so the bare-name spelling becomes unreachable and a canonicalized document gains a member nobody wrote. Also refused at import -- the test written for this commit walked straight into it. - a registry key that is not the entity's `identifier` can never resolve, so the entity is registered, validation runs, and the verdict is clean. `Context` refuses to be built that way. `tests/v3/test_extension_api.py` is the proof: a third-party codec and data type, defined and registered and judged, importing nothing private. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/docs/api/v3/entity.md | 5 + packages/zarr-metadata/mkdocs.yml | 1 + .../src/zarr_metadata/rules/_canonical.py | 12 +- .../src/zarr_metadata/rules/_documents.py | 51 +++-- .../src/zarr_metadata/rules/_v3_group.py | 16 +- .../src/zarr_metadata/v3/_entity.py | 44 +++- .../src/zarr_metadata/v3/_registry.py | 24 +++ .../zarr_metadata/v3/data_type/_families.py | 8 +- .../src/zarr_metadata/v3/entity.py | 134 ++++++++++++ .../zarr-metadata/tests/test_public_api.py | 12 ++ .../tests/v3/test_extension_api.py | 192 ++++++++++++++++++ 11 files changed, 468 insertions(+), 31 deletions(-) create mode 100644 packages/zarr-metadata/docs/api/v3/entity.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/entity.py create mode 100644 packages/zarr-metadata/tests/v3/test_extension_api.py diff --git a/packages/zarr-metadata/docs/api/v3/entity.md b/packages/zarr-metadata/docs/api/v3/entity.md new file mode 100644 index 0000000000..9ce2e1c6dd --- /dev/null +++ b/packages/zarr-metadata/docs/api/v3/entity.md @@ -0,0 +1,5 @@ +--- +title: entity +--- + +::: zarr_metadata.v3.entity diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 24b3e28240..3f84c280bd 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -25,6 +25,7 @@ nav: - ' zarr_metadata.v3.chunk_key_encoding': api/v3/chunk_key_encoding.md - ' zarr_metadata.v3.codec': api/v3/codec.md - ' zarr_metadata.v3.data_type': api/v3/data_type.md + - ' zarr_metadata.v3.entity': api/v3/entity.md - Release notes: release-notes.md # This site is a Read the Docs subproject of zarr-python; give readers a way # back to the parent docs, which list every companion package. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py index 5d0412e15a..60730ab955 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py @@ -30,7 +30,7 @@ from zarr_metadata.rules._documents import validate_array_metadata_v3 from zarr_metadata.v3._document import read_array_v3 from zarr_metadata.v3._entity import MetadataEntity -from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context if TYPE_CHECKING: from collections.abc import Mapping @@ -62,9 +62,9 @@ def __post_init__(self) -> None: raise ValueError(msg) -def _canonical_document(document: Mapping[str, object]) -> dict[str, object]: +def _canonical_document(document: Mapping[str, object], context: Context) -> dict[str, object]: """Each entity in its own canonical spelling, and the rest as written.""" - array, _ = read_array_v3(document, CORE_AND_EXTENSIONS) + array, _ = read_array_v3(document, context) out = dict(document) for key in ("data_type", "chunk_grid", "chunk_key_encoding"): entity = getattr(array, key) @@ -85,7 +85,7 @@ def _canonical_document(document: Mapping[str, object]) -> dict[str, object]: def canonicalize_array_metadata_v3( - document: ZarrV3ArrayMetadataJSON, + document: ZarrV3ArrayMetadataJSON, *, context: Context = CORE_AND_EXTENSIONS ) -> Canonical[ZarrV3ArrayMetadataJSON] | Invalid: """`document` in canonical form, or every reason it is not valid. @@ -95,14 +95,14 @@ def canonicalize_array_metadata_v3( is `Invalid` rather than a canonical document. """ normalized = cast("ZarrV3ArrayMetadataJSON", arrays_to_tuples(document)) - problems = validate_array_metadata_v3(normalized) + problems = validate_array_metadata_v3(normalized, context=context) if len(problems) != 0: return Invalid(problems) # Normalized first, so a document spelled with JSON arrays reaches the # same fixpoint as the tuple spelling. It did not: the per-field # simplifications test for `tuple`, and the validator was normalizing # on a copy the canonicalizer never saw. - canonical = _canonical_document(normalized) + canonical = _canonical_document(normalized, context) # The model layer's round trip normalizes the fields no entity owns. return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index 151e59a1e6..c753620167 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -31,7 +31,7 @@ from zarr_metadata.rules._v2_array import array_problems_v2 from zarr_metadata.rules._v3_group import group_problems_v3 from zarr_metadata.v3._document import array_problems_v3 -from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context if TYPE_CHECKING: from collections.abc import Callable @@ -51,14 +51,31 @@ def _no_semantics(document: Mapping[str, object]) -> tuple[ValidationProblem, .. return () -def _array_semantics_v3(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """The v3 array semantics, in the scope this layer chooses. +def _array_semantics_v3(context: Context) -> _SemanticValidator: + """The v3 array semantics, asked in `context`. The work is `zarr_metadata.v3` asking each entity about itself and about the parts of the document it meets; what this layer decides is which entities are in scope while it asks. """ - return array_problems_v3(document, CORE_AND_EXTENSIONS) + + def judge(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + return array_problems_v3(document, context) + + return judge + + +def _group_semantics_v3(context: Context) -> _SemanticValidator: + """The v3 group semantics, asked in `context`. + + A group's only semantic content is its inline consolidated children, + and those are array and group documents judged in the same scope. + """ + + def judge(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + return group_problems_v3(document, context) + + return judge def _judged( @@ -75,7 +92,9 @@ def _judged( return tuple(problems) -def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: +def validate_array_metadata_v3( + value: object, *, context: Context = CORE_AND_EXTENSIONS +) -> tuple[ValidationProblem, ...]: """Why `value` is not a valid v3 array document. Every structural problem, and every semantic problem that can be @@ -91,10 +110,12 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: (e.g. fresh `json.loads` output) are judged at the canonical data level rather than rejected for their spelling. """ - return _judged(arrays_to_tuples(value), _validate_structure_v3, _array_semantics_v3) + return _judged(arrays_to_tuples(value), _validate_structure_v3, _array_semantics_v3(context)) -def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: +def parse_array_metadata_v3( + value: object, *, context: Context = CORE_AND_EXTENSIONS +) -> ZarrV3ArrayMetadataJSON: """Return `value` as a valid `ZarrV3ArrayMetadataJSON`, or raise. Normalizes JSON arrays to tuples, then raises a single @@ -102,7 +123,7 @@ def parse_array_metadata_v3(value: object) -> ZarrV3ArrayMetadataJSON: problem found. """ normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_structure_v3, _array_semantics_v3) + problems = _judged(normalized, _validate_structure_v3, _array_semantics_v3(context)) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3ArrayMetadataJSON", normalized) @@ -131,20 +152,26 @@ def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: return cast("ZarrV2ArrayMetadataJSON", normalized) -def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: +def validate_group_metadata_v3( + value: object, *, context: Context = CORE_AND_EXTENSIONS +) -> tuple[ValidationProblem, ...]: """Every reason `value` is not a valid v3 group document. Composition rules recurse into inline consolidated metadata, so a consolidated child document invalid under its own rules is reported here, at its path. """ - return _judged(arrays_to_tuples(value), _validate_group_structure_v3, group_problems_v3) + return _judged( + arrays_to_tuples(value), _validate_group_structure_v3, _group_semantics_v3(context) + ) -def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: +def parse_group_metadata_v3( + value: object, *, context: Context = CORE_AND_EXTENSIONS +) -> ZarrV3GroupMetadataJSON: """Return `value` as a valid `ZarrV3GroupMetadataJSON`, or raise.""" normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_group_structure_v3, group_problems_v3) + problems = _judged(normalized, _validate_group_structure_v3, _group_semantics_v3(context)) if len(problems) != 0: raise MetadataValidationError(problems) return cast("ZarrV3GroupMetadataJSON", normalized) diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py index 99b6f39166..fa0f2c26b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py @@ -17,7 +17,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._document import array_problems_v3 -from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context if TYPE_CHECKING: from collections.abc import Sequence @@ -42,16 +42,20 @@ def _as_string_mapping(value: object) -> Mapping[str, object] | None: return cast("Mapping[str, object]", mapping) -def group_problems_v3(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: +def group_problems_v3( + document: Mapping[str, object], context: Context = CORE_AND_EXTENSIONS +) -> tuple[ValidationProblem, ...]: """Every semantic problem in a v3 group document.""" if "consolidated_metadata" not in document: return () return consolidated_entries_problems( - document["consolidated_metadata"], ("consolidated_metadata",) + document["consolidated_metadata"], ("consolidated_metadata",), context ) -def consolidated_entries_problems(value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: +def consolidated_entries_problems( + value: object, loc: Loc = (), context: Context = CORE_AND_EXTENSIONS +) -> tuple[ValidationProblem, ...]: """Semantic problems in an inline consolidated envelope's children. Structural validity of the envelope and its entries is the model @@ -72,9 +76,9 @@ def consolidated_entries_problems(value: object, loc: Loc = ()) -> tuple[Validat entry_loc = (*loc, "metadata", path) node_type = node.get("node_type") if node_type == "array": - problems.extend(_prefixed(entry_loc, array_problems_v3(node, CORE_AND_EXTENSIONS))) + problems.extend(_prefixed(entry_loc, array_problems_v3(node, context))) elif node_type == "group": - problems.extend(_prefixed(entry_loc, group_problems_v3(node))) + problems.extend(_prefixed(entry_loc, group_problems_v3(node, context))) return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 04d4ffc5dc..2baf90525e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -310,6 +310,42 @@ class MetadataEntity: required", so this is true exactly when some member is required. """ + def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: + """Refuse a subclass that forgot to say what it is. + + `identifier` and the per-kind class variables carry no default, + so a subclass omitting one type-checks cleanly and then raises + `AttributeError` from whichever method is reached first. Saying so + here makes it an import-time error in the extension's own module. + + `base=True` for a class that exists to add a class variable + rather than to be an entity -- `CodecEntity`, `IntegerDataType`. + """ + super().__init_subclass__(**kwargs) + if base: + return + missing = [name for name in cls.required_class_vars if not hasattr(cls, name)] + if len(missing) != 0: + msg = f"{cls.__name__} does not declare {', '.join(missing)}" + raise TypeError(msg) + # An optional member defaults to UNSET or `configuration` emits it + # for every instance, so the bare-name spelling becomes + # unreachable and a document gains a member it never wrote. + invented = [ + key + for key, (required, _) in cls.member_types.items() + if not required and getattr(cls, key, UNSET) is not UNSET + ] + if len(invented) != 0: + msg = ( + f"{cls.__name__} gives the optional member(s) " + f"{', '.join(invented)} a default other than UNSET" + ) + raise TypeError(msg) + + required_class_vars: ClassVar[tuple[str, ...]] = ("identifier",) + """Every class variable a concrete entity of this kind must declare.""" + @classmethod def accepts(cls, name: str) -> bool: """Whether `name` denotes this entity. @@ -409,10 +445,11 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: @dataclass(frozen=True) -class CodecEntity(MetadataEntity): +class CodecEntity(MetadataEntity, base=True): """An entity that occupies a position in the codec pipeline.""" kind: ClassVar[CodecKind] + required_class_vars: ClassVar[tuple[str, ...]] = ("identifier", "kind") variable_size: ClassVar[bool] = False """Whether this codec's output size depends on the bytes it is given. @@ -447,7 +484,7 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: @dataclass(frozen=True) -class ChunkGridEntity(MetadataEntity): +class ChunkGridEntity(MetadataEntity, base=True): """An entity that divides an array into the parts a pipeline encodes.""" def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: @@ -469,7 +506,7 @@ def grid(self, array_shape: object) -> ChunkGrid: @dataclass(frozen=True) -class DataTypeEntity(MetadataEntity): +class DataTypeEntity(MetadataEntity, base=True): """An entity that says how the array's scalars are stored. Only data types answer that, and every rule that turns on it -- a @@ -479,6 +516,7 @@ class DataTypeEntity(MetadataEntity): """ scalar_storage: ClassVar[StorageClass] + required_class_vars: ClassVar[tuple[str, ...]] = ("identifier", "scalar_storage") def storage_class(self) -> StorageClass | None: """How one scalar occupies bytes, or None if undetermined. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 48610c5f71..9cb82a613e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -94,6 +94,30 @@ class Context: entities: Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]] + def __post_init__(self) -> None: + """Refuse a table whose key an entity would not answer to. + + `resolve` finds a candidate by key and then asks the entity + whether the name is really one of its own, so a key that is not + the entity's `identifier` can never resolve. If the two disagree + -- a typo, or a rename that missed one of the two places the name + is written -- registration appears to succeed, validation runs, + and the verdict is clean. Indistinguishable from extension + openness, and the easiest way to ship a broken extension. + + The key is the identifier, not a name a document writes: the + raw-bytes family registers under an invented one that `accepts` + deliberately refuses. + """ + for field, table in self.entities.items(): + for key, entity in table.items(): + if key != entity.identifier: + msg = ( + f"{entity.__name__} is registered at {field!r} under {key!r} " + f"but its identifier is {entity.identifier!r}" + ) + raise ValueError(msg) + def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: """The entity `name` denotes at `field`, or None if out of scope. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 5364dfacfd..94205d71e7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -63,7 +63,7 @@ def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[Validati @dataclass(frozen=True) -class IntegerDataType(DataTypeEntity): +class IntegerDataType(DataTypeEntity, base=True): """A fixed-width integer. The width is the whole difference.""" bounds: ClassVar[tuple[int, int]] @@ -80,7 +80,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class FloatDataType(DataTypeEntity): +class FloatDataType(DataTypeEntity, base=True): """A binary float. A fill value may be a number, a named non-finite, or hex.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -120,7 +120,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class ComplexDataType(DataTypeEntity): +class ComplexDataType(DataTypeEntity, base=True): """A complex number: a `[real, imag]` pair of the component float type.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -139,7 +139,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class NumpyTimeDataType(DataTypeEntity): +class NumpyTimeDataType(DataTypeEntity, base=True): """A numpy time scalar: a signed 64-bit count of units, or `NaT`.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py new file mode 100644 index 0000000000..b545057d3c --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -0,0 +1,134 @@ +"""The extension layer: what an entity is, and what is in scope. + +Every Zarr v3 extension point -- codecs, data types, chunk grids, chunk +key encodings, storage transformers -- is modelled as a class that +answers for itself. This module is the public door to that layer, for two +kinds of caller. + +**Reading metadata.** The concrete entities know things a document does +not spell out: what a data type's scalars are, which position a codec +occupies in the pipeline, what a chunk grid divides an array into. Reach +them through a scope: + + from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS + + data_type, problems = CORE_AND_EXTENSIONS.coerce("data_type", "int32") + data_type.storage_class() # 'multi_byte' + +**Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, +`ChunkGridEntity` or `MetadataEntity`, declare `identifier` and +`member_types`, and put it in a `Context`: + + @dataclass(frozen=True) + class AcmeLz4Codec(CodecEntity): + acceleration: int = 1 + + identifier: ClassVar[str] = "acme.lz4" + kind: ClassVar[CodecKind] = "bytes_bytes" + member_types: ClassVar[MemberTypes] = {"acceleration": (False, is_int)} + + SCOPE = Context({**CORE_AND_EXTENSIONS.entities, + "codecs": {**CORE_AND_EXTENSIONS.entities["codecs"], + AcmeLz4Codec.identifier: AcmeLz4Codec}}) + + validate_array_metadata_v3(document, context=SCOPE) + +A name in no scope is not rejected -- that is what extension openness +means -- so registering yours is how you get it judged rather than waved +through. +""" + +from __future__ import annotations + +from zarr_metadata.v3._chain import chain_problems, order_problems +from zarr_metadata.v3._document import ArrayDocumentV3, array_problems_v3, read_array_v3 +from zarr_metadata.v3._entity import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, + STORAGE_TRANSFORMERS, + ChunkGridEntity, + CodecEntity, + CodecKind, + Coerced, + DataTypeEntity, + ExtensionPointField, + Loc, + MemberTypes, + MetadataEntity, + StorageClass, + TypeCheck, + coerce_members, + is_bool, + is_int, + is_integer, + is_json_value, + is_str, + named_configuration, + one_of, + problem, + sequence_of, + within, +) +from zarr_metadata.v3._parts import UNKNOWN_GRID, ArrayParts, ChunkGrid, Extents, shard_index_grid +from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context +from zarr_metadata.v3.data_type._families import ( + FLOAT_SPECIALS, + ComplexDataType, + FloatDataType, + IntegerDataType, + NumpyTimeDataType, + as_sequence, + byte_values, +) + +__all__ = [ + "CHUNK_GRID", + "CHUNK_KEY_ENCODING", + "CODECS", + "CORE", + "CORE_AND_EXTENSIONS", + "DATA_TYPE", + "FLOAT_SPECIALS", + "STORAGE_TRANSFORMERS", + "UNKNOWN_GRID", + "ArrayDocumentV3", + "ArrayParts", + "ChunkGrid", + "ChunkGridEntity", + "CodecEntity", + "CodecKind", + "Coerced", + "ComplexDataType", + "Context", + "DataTypeEntity", + "ExtensionPointField", + "Extents", + "FloatDataType", + "IntegerDataType", + "Loc", + "MemberTypes", + "MetadataEntity", + "NumpyTimeDataType", + "StorageClass", + "TypeCheck", + "array_problems_v3", + "as_sequence", + "byte_values", + "chain_problems", + "coerce_members", + "is_bool", + "is_int", + "is_integer", + "is_json_value", + "is_str", + "named_configuration", + "one_of", + "order_problems", + "problem", + "read_array_v3", + "sequence_of", + "shard_index_grid", + "within", +] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 5858cc375b..266bbaf77d 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -271,6 +271,7 @@ def test_all_is_grouped_and_unique() -> None: # the extension's type checks, value checks and canonical spelling. # Listed last so a longer role still wins the alternation. "Codec", + "Entity", "ChunkGrid", "ChunkKeyEncoding", "DataType", @@ -290,6 +291,17 @@ def test_all_is_grouped_and_unique() -> None: "CastOutOfRangeMode", "CastRoundingMode", "CodecKind", + "TypeCheck", + "StorageClass", + "MemberTypes", + "Loc", + "Extents", + "ExtensionPointField", + "Context", + "Coerced", + "ChunkGrid", + "ArrayParts", + "ArrayDocumentV3", "Endianness", "Invalid", "HexFloat16", diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py new file mode 100644 index 0000000000..55e42f752f --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -0,0 +1,192 @@ +"""A third party registering its own entity, through public API only. + +Every import here is from a module without a leading underscore. If this +file has to reach into a private one, the extension surface is not real. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar + +import pytest + +from zarr_metadata.model import UNSET, ValidationProblem +from zarr_metadata.rules import ( + canonicalize_array_metadata_v3, + validate_array_metadata_v3, +) +from zarr_metadata.v3.entity import ( + CORE_AND_EXTENSIONS, + ArrayParts, + CodecEntity, + CodecKind, + Context, + DataTypeEntity, + MemberTypes, + MetadataEntity, + StorageClass, + is_int, + problem, +) + +ACME_MAX_ACCELERATION = 65537 + + +@dataclass(frozen=True) +class AcmeLz4Codec(CodecEntity): + """A third-party compressor.""" + + acceleration: int | UNSET = UNSET + + identifier: ClassVar[str] = "acme.lz4" + kind: ClassVar[CodecKind] = "bytes_bytes" + variable_size: ClassVar[bool] = True + member_types: ClassVar[MemberTypes] = {"acceleration": (False, is_int)} + + def problems(self) -> tuple[ValidationProblem, ...]: + if self.acceleration is UNSET: + return () + if not 1 <= self.acceleration <= ACME_MAX_ACCELERATION: + return problem( + ("acceleration",), + f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {self.acceleration}", + "invalid_value", + ) + return () + + +@dataclass(frozen=True) +class AcmeFloat8DataType(DataTypeEntity): + """A third-party one-byte float.""" + + identifier: ClassVar[str] = "acme.float8" + scalar_storage: ClassVar[StorageClass] = "single_byte" + + +def _scope() -> Context: + entities = dict(CORE_AND_EXTENSIONS.entities) + return Context( + { + **entities, + "codecs": {**entities["codecs"], AcmeLz4Codec.identifier: AcmeLz4Codec}, + "data_type": { + **entities["data_type"], + AcmeFloat8DataType.identifier: AcmeFloat8DataType, + }, + } + ) + + +SCOPE = _scope() + + +def _document(**overrides: object) -> dict[str, object]: + return { + "zarr_format": 3, + "node_type": "array", + "shape": (8,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (8,)}}, + "chunk_key_encoding": "default", + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + **overrides, + } + + +def test_an_unregistered_name_is_not_judged() -> None: + # Extension openness: out of scope means unjudged, not invalid. + document = _document( + codecs=( + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "acme.lz4", "configuration": {"acceleration": 999999}}, + ) + ) + assert validate_array_metadata_v3(document) == () # type: ignore[arg-type] + + +def test_a_registered_entity_is_judged() -> None: + document = _document( + codecs=( + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "acme.lz4", "configuration": {"acceleration": 999999}}, + ) + ) + problems = validate_array_metadata_v3(document, context=SCOPE) # type: ignore[arg-type] + assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "acceleration")] + + +def test_a_registered_entity_joins_the_pipeline_rules() -> None: + # Declared `bytes_bytes`, so it may not precede the array->bytes codec, + # and it is variable-size, so it may not encode a shard index. + document = _document( + codecs=("acme.lz4", {"name": "bytes", "configuration": {"endian": "little"}}) + ) + problems = validate_array_metadata_v3(document, context=SCOPE) # type: ignore[arg-type] + assert [problem.loc for problem in problems] == [("codecs", 1)] + + +def test_a_registered_data_type_drives_the_codecs_around_it() -> None: + # Single-byte, so the `bytes` codec needs no endianness for it. + document = _document(data_type="acme.float8", fill_value=0, codecs=("bytes",)) + assert validate_array_metadata_v3(document, context=SCOPE) == () # type: ignore[arg-type] + + +def test_a_registered_entity_canonicalizes_itself() -> None: + document = _document( + codecs=( + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "acme.lz4", "configuration": {}}, + ) + ) + result = canonicalize_array_metadata_v3(document, context=SCOPE) # type: ignore[arg-type] + assert result.valid is True + assert result.document["codecs"][1] == "acme.lz4" # type: ignore[index] + + +def test_error_an_entity_must_say_what_it_is() -> None: + with pytest.raises(TypeError, match="does not declare identifier"): + + @dataclass(frozen=True) + class Nameless(CodecEntity): + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_error_a_registry_key_must_be_the_identifier() -> None: + # Otherwise `resolve` never finds it and the document is silently + # waved through, indistinguishable from openness. + with pytest.raises(ValueError, match="registered at 'codecs' under 'acme.lz-4'"): + Context({"codecs": {"acme.lz-4": AcmeLz4Codec}}) + + +def test_the_entity_layer_answers_what_a_reader_needs() -> None: + # The questions zarr-python asks before it can read a chunk. + data_type, problems = CORE_AND_EXTENSIONS.coerce("data_type", "int32") + assert problems == () + assert isinstance(data_type, DataTypeEntity) + assert data_type.storage_class() == "multi_byte" + + grid, problems = CORE_AND_EXTENSIONS.coerce( + "chunk_grid", {"name": "regular", "configuration": {"chunk_shape": (32, 32)}} + ) + assert problems == () + assert isinstance(grid, MetadataEntity) + parts = ArrayParts(grid.grid((64, 64)), data_type) # type: ignore[attr-defined] + assert parts.grid.rank == 2 + assert parts.grid.axis(0) == frozenset({32}) + + +def test_error_an_optional_member_defaults_to_unset() -> None: + # Otherwise every instance emits it, the bare-name spelling becomes + # unreachable, and a canonicalized document gains a member the writer + # never wrote. + with pytest.raises(TypeError, match="a default other than UNSET"): + + @dataclass(frozen=True) + class Inventive(CodecEntity): + level: int = 3 + + identifier: ClassVar[str] = "acme.inventive" + kind: ClassVar[CodecKind] = "bytes_bytes" + member_types: ClassVar[MemberTypes] = {"level": (False, is_int)} From cb4e2b08d6896169a341f0f694c9d69e959af4ca Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:35:34 +0200 Subject: [PATCH 040/107] docs(zarr-metadata): record what the review changed, and what it did not Includes the mypy assignability question, which was investigated and closed rather than fixed: an entity's `to_json` returns its own object TypedDict, mypy will not accept that where the envelope type is wanted, and every widening that satisfies mypy costs more than it buys -- `object` stops the package's `st.from_type` strategies terminating, and `Any` stops the corrupting strategy generating the ill-typed members it exists to generate, which its own reach test caught. The narrow type is also the true one. `zarr_metadata.v3.entity` documents the `cast`. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.misc.2.md | 32 +++++++++++++++++++ .../src/zarr_metadata/v3/entity.py | 11 +++++++ 2 files changed, 43 insertions(+) create mode 100644 packages/zarr-metadata/changes/4379.misc.2.md diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md new file mode 100644 index 0000000000..1eb8836eac --- /dev/null +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -0,0 +1,32 @@ +Adversarial review of the entity layer, from five directions: reading and +writing `zarr.json` the way zarr-python does, authoring a third-party +extension, hunting wrong verdicts, type-checking a strict consumer, and +v2/group/consolidated workloads. Eleven real defects, all reproduced, all +fixed here; the rest of what it found is recorded against the merged +model layer instead. + +The one that mattered most: a metadata field nested inside a +configuration -- a shard's pipelines, a struct field's data type, a +`cast_value` target -- was not getting the structural judgment a +top-level one gets, so an extra member, a non-object `configuration` or a +non-boolean `must_understand` was accepted there and then *deleted* by +the canonicalizer. The differential over 2267 documents could not see it, +because `st.from_type` honours the TypedDicts and cannot emit a malformed +envelope. + +Two results worth recording. Eleven `zarr.json` documents written by this +repository's own `create_array` -- float32+blosc, sharded, string, +datetime64, complex128, structured, transposed -- validate clean. And +against the rule registry this layer replaced, over one shared corpus of +800 documents, no verdict differs in the laxer direction and the valid +documents report identically. + +One thing deliberately not done. An entity's `to_json` returns its own +object TypedDict, which mypy will not accept where a +`ZarrV3MetadataFieldJSON` is wanted, because it reads a TypedDict as +`Mapping[str, object]` and never as the `Mapping[str, JSONValue]` the +envelope declares. Widening the envelope fixes mypy and costs more than +it buys: `object` stops the package's own `from_type` strategies +terminating, and `Any` stops them generating the ill-typed members they +exist to generate. The narrow type is also the true one, so it stays, and +`zarr_metadata.v3.entity` says so. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index b545057d3c..dab579aaa8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -36,6 +36,17 @@ class AcmeLz4Codec(CodecEntity): A name in no scope is not rejected -- that is what extension openness means -- so registering yours is how you get it judged rather than waved through. + +One known friction, under mypy only. An entity's `to_json` returns its own +object TypedDict, and mypy does not accept that where a +`ZarrV3MetadataFieldJSON` is wanted -- it reads a TypedDict as +`Mapping[str, object]` and never as the `Mapping[str, JSONValue]` the +envelope declares. Putting `to_json()` output straight into a `codecs` +list therefore needs a `cast` under mypy; pyright accepts it. Widening the +envelope fixes mypy and costs more than it buys: with `object` the +package's own `st.from_type` strategies stop terminating, and with `Any` +they stop generating the ill-typed members they exist to generate. The +narrow type is also the true one -- a configuration's values are JSON. """ from __future__ import annotations From 0500a37d23aeda7154f831849d31417fffa12002 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:42:49 +0200 Subject: [PATCH 041/107] docs(zarr-metadata): the narrow configuration type is sound, not a compromise The previous wording justified keeping `Mapping[str, JSONValue]` by what widening did to this package's Hypothesis strategies. That is backwards: a test utility's behaviour is a fact about the utility, not an argument about a type. The real reason is that the conversion mypy refuses is sound here. Mypy's rule exists because an ordinary TypedDict may carry extra items of undeclared types, so the union of declared value types does not bound the mapping. Every TypedDict in this package is `closed` (PEP 728), which forbids that; pyright implements PEP 728 and accepts the assignment, and mypy has not implemented it yet (python/mypy#8994, python/mypy#18439). So the annotation is accurate and stays accurate, one checker needs a `cast` until it catches up, and widening would buy mypy's silence by making the type say something false. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.misc.2.md | 19 ++++++++------ .../src/zarr_metadata/v3/entity.py | 25 +++++++++++++------ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 1eb8836eac..4e8d38e9a3 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -23,10 +23,15 @@ documents report identically. One thing deliberately not done. An entity's `to_json` returns its own object TypedDict, which mypy will not accept where a -`ZarrV3MetadataFieldJSON` is wanted, because it reads a TypedDict as -`Mapping[str, object]` and never as the `Mapping[str, JSONValue]` the -envelope declares. Widening the envelope fixes mypy and costs more than -it buys: `object` stops the package's own `from_type` strategies -terminating, and `Any` stops them generating the ill-typed members they -exist to generate. The narrow type is also the true one, so it stays, and -`zarr_metadata.v3.entity` says so. +`ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as +`Mapping[str, object]`, never as the `Mapping[str, JSONValue]` the +envelope declares, so the write path needs a `cast` under mypy. + +The conversion is sound and the annotation stays. Mypy's rule exists +because an ordinary TypedDict may carry extra items of undeclared types, +so the union of the declared value types does not bound the mapping; +every TypedDict here is `closed` (PEP 728), which forbids that, and +pyright implements PEP 728 while mypy does not yet (python/mypy#8994, +python/mypy#18439). Widening `configuration` would satisfy mypy by making +the annotation say something false, and a configuration's values are +JSON. `zarr_metadata.v3.entity` records the `cast` and the reason. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index dab579aaa8..c6f3d11790 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -39,14 +39,23 @@ class AcmeLz4Codec(CodecEntity): One known friction, under mypy only. An entity's `to_json` returns its own object TypedDict, and mypy does not accept that where a -`ZarrV3MetadataFieldJSON` is wanted -- it reads a TypedDict as -`Mapping[str, object]` and never as the `Mapping[str, JSONValue]` the -envelope declares. Putting `to_json()` output straight into a `codecs` -list therefore needs a `cast` under mypy; pyright accepts it. Widening the -envelope fixes mypy and costs more than it buys: with `object` the -package's own `st.from_type` strategies stop terminating, and with `Any` -they stop generating the ill-typed members they exist to generate. The -narrow type is also the true one -- a configuration's values are JSON. +`ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as +`Mapping[str, object]`, never as the `Mapping[str, JSONValue]` the +envelope declares. So putting `to_json()` output straight into a `codecs` +list needs a `cast` under mypy. Pyright accepts it. + +That conversion is sound here, which is why pyright is the one that is +right. The rule mypy is applying exists because an ordinary TypedDict may +carry extra items of types it never declared, so the union of the +declared value types does not bound what is in the mapping. Every +TypedDict in this package is `closed` (PEP 728), which forbids exactly +that, and pyright implements PEP 728. Mypy does not yet -- see +python/mypy#8994 and python/mypy#18439. + +The type therefore stays as it is. Widening `configuration` to +`Mapping[str, object]` or `Mapping[str, Any]` would satisfy mypy by +making the annotation say something false: a configuration's values are +JSON, and that is worth more than one checker's `cast`. """ from __future__ import annotations From 457ca1b73dfc34f41885b9d441f6b1375fa2a7c8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 18:49:58 +0200 Subject: [PATCH 042/107] fix(zarr-metadata): silence unused-class on the two guard tests `just typecheck` pins pyright to 1.1.404 to match CI; a newer one does not report `reportUnusedClass` here, which is why this passed locally and failed on the runner. Both classes exist to be refused while they are being created, so neither is ever bound and neither can be used. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/tests/v3/test_extension_api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 55e42f752f..18c85d127a 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -147,9 +147,10 @@ def test_a_registered_entity_canonicalizes_itself() -> None: def test_error_an_entity_must_say_what_it_is() -> None: with pytest.raises(TypeError, match="does not declare identifier"): - + # Never bound: the guard raises while the class is being created, + # which is the whole point -- so pyright cannot see it used. @dataclass(frozen=True) - class Nameless(CodecEntity): + class Nameless(CodecEntity): # pyright: ignore[reportUnusedClass] kind: ClassVar[CodecKind] = "bytes_bytes" @@ -184,7 +185,7 @@ def test_error_an_optional_member_defaults_to_unset() -> None: with pytest.raises(TypeError, match="a default other than UNSET"): @dataclass(frozen=True) - class Inventive(CodecEntity): + class Inventive(CodecEntity): # pyright: ignore[reportUnusedClass] level: int = 3 identifier: ClassVar[str] = "acme.inventive" From 2c9e0fae2a485129315f8ac1b58d8396339aba49 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 19:21:57 +0200 Subject: [PATCH 043/107] feat(zarr-metadata): a real union at every extension point, and a reader's door `MetadataEntity | object` was fifteen annotations that said something to a reader and nothing to a checker: `X | object` is `object`. Consumers paid two isinstance calls per field, the first useless, and could not distinguish an extension this package does not model from one that was modelled and refused -- both arrived as the raw value. `Opaque` carries the JSON and says which it is. Every extension point is now an exhaustive two-case union naming its own kind, and `Context.coerce` is overloaded on the extension point so it returns that kind rather than the base. The extension-point constants lose their widening annotation to keep their `Literal` types, which is what makes the overloads fire at call sites written with them. `ArrayDocumentV3.from_json` is the fail-fast reader zarrs has and this did not: construct or raise, with every problem on the exception. An out-of-scope name does not raise -- refusing it would make extension openness unimplementable -- so what fails is metadata that is wrong, not metadata that is unfamiliar. The judgment moves onto `ArrayDocumentV3.problems()`, so a document is read once whichever door you come in by. Verdicts are unchanged: the 800-document differential against the rule registry reports exactly as before. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.9.md | 21 ++++ .../src/zarr_metadata/rules/_canonical.py | 2 +- .../src/zarr_metadata/v3/_document.py | 103 +++++++++++++----- .../src/zarr_metadata/v3/_entity.py | 35 +++++- .../src/zarr_metadata/v3/_registry.py | 60 +++++++++- .../src/zarr_metadata/v3/codec/cast_value.py | 14 ++- .../v3/codec/sharding_indexed.py | 20 ++-- .../src/zarr_metadata/v3/data_type/struct.py | 6 +- .../src/zarr_metadata/v3/entity.py | 29 +++-- .../zarr-metadata/tests/test_public_api.py | 1 + .../tests/v3/test_extension_api.py | 66 ++++++++++- 11 files changed, 292 insertions(+), 65 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.feature.9.md diff --git a/packages/zarr-metadata/changes/4379.feature.9.md b/packages/zarr-metadata/changes/4379.feature.9.md new file mode 100644 index 0000000000..16bf298c17 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.feature.9.md @@ -0,0 +1,21 @@ +`ArrayDocumentV3.from_json` is the fail-fast front door for readers: one +call, and either every extension point is read into an entity or a single +`MetadataValidationError` carries every reason it is not, structural and +semantic together. `validate_array_metadata_v3` remains for callers who +want the problems as data. + +A name the scope does not model is not a failure. It arrives as an +`Opaque` carrying the JSON the document wrote and saying which kind of +not-an-entity it is -- `out_of_scope` for an extension this reader does +not know, which is the reader's cue to resolve it elsewhere, and +`invalid` for a name that was claimed and then refused. + +That distinction was previously impossible to make, because every +extension point was typed `MetadataEntity | object` -- which *is* +`object`, so the union narrowed to nothing and a consumer could not tell +a third-party codec from a malformed one. Each field now names its own +kind (`DataTypeEntity | Opaque`, `CodecEntity | Opaque`, ...), an +exhaustive two-case union that narrows. `Context.coerce` is overloaded on +the extension point, so it returns the entity type for that point rather +than the base, and the extension-point constants keep their `Literal` +types so a call written with one of them gets the narrow result. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py index 60730ab955..3f0870e9ff 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py @@ -72,7 +72,7 @@ def _canonical_document(document: Mapping[str, object], context: Context) -> dic out[key] = entity.to_json() if "codecs" in out: out["codecs"] = tuple( - codec.to_json() if isinstance(codec, MetadataEntity) else codec + codec.to_json() if isinstance(codec, MetadataEntity) else codec.json for codec in array.codecs ) names = out.get("dimension_names") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index a93333809c..5632639838 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -16,10 +16,18 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Final, cast -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import ( + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, +) +from zarr_metadata.model._validation import ( + validate_array_metadata_v3 as validate_array_metadata_v3_structure, +) from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._entity import ( CHUNK_GRID, @@ -28,33 +36,79 @@ DATA_TYPE, STORAGE_TRANSFORMERS, ChunkGridEntity, + CodecEntity, DataTypeEntity, ExtensionPointField, MetadataEntity, + Opaque, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid +from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context if TYPE_CHECKING: - from collections.abc import Mapping, Sequence - - from zarr_metadata.v3._registry import Context + from collections.abc import Sequence @dataclass(frozen=True, slots=True) class ArrayDocumentV3: """A v3 array document with its extension points read as entities. - A field holds the value untouched where its name was out of scope, so - an unmodelled extension survives the reading and is simply not judged. + A field that could not be read holds an `Opaque`, which carries the + JSON the document wrote and says whether the name was out of scope -- + an extension this reader does not model, which is not an error -- or + claimed and refused. Both are narrowable: every field is an exhaustive + two-case union. """ document: Mapping[str, object] - data_type: MetadataEntity | object - chunk_grid: MetadataEntity | object - chunk_key_encoding: MetadataEntity | object - codecs: tuple[MetadataEntity | object, ...] - storage_transformers: tuple[MetadataEntity | object, ...] + data_type: DataTypeEntity | Opaque + chunk_grid: ChunkGridEntity | Opaque + chunk_key_encoding: MetadataEntity | Opaque + codecs: tuple[CodecEntity | Opaque, ...] + storage_transformers: tuple[MetadataEntity | Opaque, ...] + + def problems(self) -> tuple[ValidationProblem, ...]: + """Every semantic problem this document has, once it has been read. + + The type-space problems are `read_array_v3`'s, because they are + the reasons some of this is `Opaque` rather than an entity. + """ + return ( + *_entity_problems(self), + *_fill_value_problems(self), + *_grid_problems(self), + *_dimension_names_problems(self), + *chain_problems(self.codecs, self.parts, ("codecs",)), + ) + + @classmethod + def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> ArrayDocumentV3: + """A v3 array document read into entities, or raise. + + The reader's front door, and the one entry point that fails fast: + one call, and either every extension point is read or a single + `MetadataValidationError` carries every reason it is not -- + structural and semantic together. Use `validate_array_metadata_v3` + instead when you want the problems as data. + + A name this `context` does not model is *not* a failure. It comes + back as an `Opaque` marked `out_of_scope`, because a document may + legitimately use an extension this reader does not know, and + refusing it would make openness unimplementable. What fails is + metadata that is wrong, not metadata that is unfamiliar. + """ + normalized = arrays_to_tuples(value) + problems = validate_array_metadata_v3_structure(normalized) + if isinstance(normalized, Mapping) and len(problems) == 0: + document = cast("Mapping[str, object]", normalized) + array, found = read_array_v3(document, context) + problems = (*found, *array.problems()) + if len(problems) == 0: + return array + if len(problems) == 0: # pragma: no cover - a non-mapping always has problems + problems = (ValidationProblem((), "expected a v3 array document", "invalid_type"),) + raise MetadataValidationError(problems) @property def parts(self) -> ArrayParts: @@ -98,19 +152,19 @@ def read_array_v3( Type-space only: what comes back is well-typed by construction, and the problems are the reasons some of it is not an entity. """ - read: dict[str, MetadataEntity | object] = {} + read: dict[str, MetadataEntity | Opaque] = {} problems: list[ValidationProblem] = [] for field, key in _SINGLE_FIELDS: value = document.get(key) if value is None: - read[key] = None + read[key] = Opaque(None, "invalid") continue entity, found = context.coerce(field, value, (key,), envelope_judged=True) read[key] = entity problems.extend(found) - sequences: dict[str, tuple[MetadataEntity | object, ...]] = {} + sequences: dict[str, tuple[MetadataEntity | Opaque, ...]] = {} for field, key in _SEQUENCE_FIELDS: - read_entries: list[MetadataEntity | object] = [] + read_entries: list[MetadataEntity | Opaque] = [] entries = document.get(key) if isinstance(entries, (list, tuple)): for index, entry in enumerate(cast("Sequence[object]", entries)): @@ -121,10 +175,10 @@ def read_array_v3( return ( ArrayDocumentV3( document=document, - data_type=read["data_type"], - chunk_grid=read["chunk_grid"], + data_type=cast("DataTypeEntity | Opaque", read["data_type"]), + chunk_grid=cast("ChunkGridEntity | Opaque", read["chunk_grid"]), chunk_key_encoding=read["chunk_key_encoding"], - codecs=sequences["codecs"], + codecs=cast("tuple[CodecEntity | Opaque, ...]", sequences["codecs"]), storage_transformers=sequences["storage_transformers"], ), tuple(problems), @@ -139,7 +193,9 @@ def _entity_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: if isinstance(entity, MetadataEntity): found.extend(within((key,), entity.problems())) for _, key in _SEQUENCE_FIELDS: - for index, entity in enumerate(cast("tuple[object, ...]", getattr(array, key))): + for index, entity in enumerate( + cast("tuple[MetadataEntity | Opaque, ...]", getattr(array, key)) + ): if isinstance(entity, MetadataEntity): found.extend(within((key, index), entity.problems())) return tuple(found) @@ -187,14 +243,7 @@ def array_problems_v3( member is present and typed as its TypedDict declares. """ array, problems = read_array_v3(document, context) - return ( - *problems, - *_entity_problems(array), - *_fill_value_problems(array), - *_grid_problems(array), - *_dimension_names_problems(array), - *chain_problems(array.codecs, array.parts, ("codecs",)), - ) + return (*problems, *array.problems()) __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 2baf90525e..8f0926ca5a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -88,11 +88,15 @@ data types, which import this. """ -DATA_TYPE: Final[ExtensionPointField] = "data_type" -CHUNK_GRID: Final[ExtensionPointField] = "chunk_grid" -CHUNK_KEY_ENCODING: Final[ExtensionPointField] = "chunk_key_encoding" -CODECS: Final[ExtensionPointField] = "codecs" -STORAGE_TRANSFORMERS: Final[ExtensionPointField] = "storage_transformers" +# Left to infer their `Literal` types rather than widened to +# `ExtensionPointField`: `Context.coerce` overloads on the field, so a +# call written with one of these constants gets the entity type back +# rather than the base. They are still assignable to the alias. +DATA_TYPE: Final = "data_type" +CHUNK_GRID: Final = "chunk_grid" +CHUNK_KEY_ENCODING: Final = "chunk_key_encoding" +CODECS: Final = "codecs" +STORAGE_TRANSFORMERS: Final = "storage_transformers" StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How one scalar of a data type occupies bytes. @@ -250,6 +254,26 @@ def coerce_members( return members, tuple(problems), frozenset(unreadable) +@dataclass(frozen=True, slots=True) +class Opaque: + """A metadata field this reading did not turn into an entity. + + Carrying the JSON rather than dropping it is what makes the result a + real union: `CodecEntity | Opaque` is exhaustive and narrows, where + `CodecEntity | object` is just `object` and narrows to nothing. + + `reason` is the distinction a reader needs and could not otherwise + make. `out_of_scope` is a name no entity in this `Context` claims -- + an extension this reader does not model, which is not an error and is + the reader's cue to resolve it elsewhere. `invalid` is a name that + *was* claimed and then refused; the reasons are in the problems + reported alongside. + """ + + json: object + reason: Literal["out_of_scope", "invalid"] + + # No `slots=True`, deliberately. It rebuilds the class, which on Python # 3.11 and 3.12 leaves the zero-argument `super()` *in that same class's # body* pointing at the class it replaced. Several entities call `super()` @@ -597,6 +621,7 @@ def named_configuration( "Loc", "MemberTypes", "MetadataEntity", + "Opaque", "StorageClass", "TypeCheck", "coerce_members", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 9cb82a613e..cb423975e4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -21,13 +21,19 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Literal, overload from zarr_metadata.model._validation import ( ValidationProblem, validate_metadata_field_v3, ) -from zarr_metadata.v3._entity import named_configuration +from zarr_metadata.v3._entity import ( + ChunkGridEntity, + CodecEntity, + DataTypeEntity, + Opaque, + named_configuration, +) from zarr_metadata.v3._extension_points import ( CHUNK_GRID, CHUNK_KEY_ENCODING, @@ -135,6 +141,46 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] return None return entity + @overload + def coerce( + self, + field: Literal["data_type"], + value: object, + loc: Loc = (), + *, + envelope_judged: bool = False, + ) -> tuple[DataTypeEntity | Opaque, tuple[ValidationProblem, ...]]: ... + + @overload + def coerce( + self, + field: Literal["codecs"], + value: object, + loc: Loc = (), + *, + envelope_judged: bool = False, + ) -> tuple[CodecEntity | Opaque, tuple[ValidationProblem, ...]]: ... + + @overload + def coerce( + self, + field: Literal["chunk_grid"], + value: object, + loc: Loc = (), + *, + envelope_judged: bool = False, + ) -> tuple[ChunkGridEntity | Opaque, tuple[ValidationProblem, ...]]: ... + + @overload + def coerce( + self, + field: ExtensionPointField, + value: object, + loc: Loc = (), + *, + envelope_judged: bool = False, + ) -> tuple[MetadataEntity | Opaque, tuple[ValidationProblem, ...]]: ... + def coerce( self, field: ExtensionPointField, @@ -142,7 +188,7 @@ def coerce( loc: Loc = (), *, envelope_judged: bool = False, - ) -> tuple[MetadataEntity | object, tuple[ValidationProblem, ...]]: + ) -> tuple[MetadataEntity | Opaque, tuple[ValidationProblem, ...]]: """One nested entity, read in this scope. The primitive the containing entities are built from: a `struct` @@ -169,18 +215,20 @@ def coerce( ) name, _, _ = named_configuration(value) if name is None: - return value, ( + return Opaque(value, "invalid"), ( *problems, ValidationProblem(loc, f"expected a metadata field, got {value!r}", "invalid_type"), ) entity_type = self.resolve(field, name) if entity_type is None: - return value, tuple(problems) + return Opaque(value, "out_of_scope"), tuple(problems) entity, found = entity_type.coerce(value, self) problems.extend( ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found ) - return (value if entity is None else entity), tuple(problems) + if entity is None: + return Opaque(value, "invalid"), tuple(problems) + return entity, tuple(problems) _CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index ed2a99368a..a7f9bb3a49 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -18,7 +18,7 @@ DataTypeEntity, Loc, MemberTypes, - MetadataEntity, + Opaque, is_json_value, one_of, problem, @@ -176,6 +176,10 @@ def _is_data_type_field(value: object, loc: Loc) -> tuple[ValidationProblem, ... return () +_UNREAD: Final = Opaque(None, "invalid") +"""Placeholder for `data_type`, which is required and so never defaulted.""" + + @dataclass(frozen=True) class CastValueCodec(CodecEntity): """The `cast_value` codec, coerced from its metadata. @@ -184,7 +188,7 @@ class CastValueCodec(CodecEntity): read in a scope rather than on its own. """ - data_type: MetadataEntity | object = None + data_type: DataTypeEntity | Opaque = _UNREAD rounding: CastRoundingMode | UNSET = UNSET out_of_range: CastOutOfRangeMode | UNSET = UNSET scalar_map: ScalarMap | UNSET = UNSET @@ -212,7 +216,7 @@ def coerce(cls, value: object, context: "Context") -> Coerced[Self]: def problems(self) -> tuple[ValidationProblem, ...]: """Whatever the data type being cast to says about itself.""" - if not isinstance(self.data_type, MetadataEntity): + if not isinstance(self.data_type, DataTypeEntity): return () return within(("data_type",), self.data_type.problems()) @@ -220,8 +224,10 @@ def configuration(self) -> dict[str, object]: """The target data type in its canonical spelling.""" members = super().configuration() data_type = self.data_type - if isinstance(data_type, MetadataEntity): + if isinstance(data_type, DataTypeEntity): members["data_type"] = data_type.to_json() + else: + members["data_type"] = data_type.json return members def transition(self, incoming: ArrayParts) -> ArrayParts | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 1aabeb4701..a7ff40c4d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -18,7 +18,7 @@ Coerced, Loc, MemberTypes, - MetadataEntity, + Opaque, is_int, one_of, problem, @@ -122,9 +122,9 @@ def _is_field_tuple(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: def _coerce_pipeline( entries: tuple[object, ...], context: "Context", loc: Loc -) -> tuple[tuple[MetadataEntity | object, ...], tuple[ValidationProblem, ...]]: +) -> tuple[tuple[CodecEntity | Opaque, ...], tuple[ValidationProblem, ...]]: """Every entry of one pipeline, read in `context`.""" - coerced: list[MetadataEntity | object] = [] + coerced: list[CodecEntity | Opaque] = [] problems: list[ValidationProblem] = [] for index, entry in enumerate(entries): codec, found = context.coerce(CODECS, entry, (*loc, index)) @@ -143,8 +143,8 @@ class ShardingIndexedCodec(CodecEntity): """ chunk_shape: tuple[int, ...] = () - codecs: tuple[MetadataEntity | object, ...] = () - index_codecs: tuple[MetadataEntity | object, ...] = () + codecs: tuple[CodecEntity | Opaque, ...] = () + index_codecs: tuple[CodecEntity | Opaque, ...] = () index_location: ShardingIndexLocation | UNSET = UNSET identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME @@ -190,8 +190,10 @@ def problems(self) -> tuple[ValidationProblem, ...]: if extent < 1 ] for member in ("codecs", "index_codecs"): - for position, codec in enumerate(cast("tuple[object, ...]", getattr(self, member))): - if isinstance(codec, MetadataEntity): + for position, codec in enumerate( + cast("tuple[CodecEntity | Opaque, ...]", getattr(self, member)) + ): + if isinstance(codec, CodecEntity): found.extend(within((member, position), codec.problems())) return tuple(found) @@ -272,8 +274,8 @@ def configuration(self) -> dict[str, object]: members = super().configuration() for member in ("codecs", "index_codecs"): members[member] = tuple( - entry.to_json() if isinstance(entry, MetadataEntity) else entry - for entry in cast("tuple[object, ...]", members[member]) + entry.to_json() if isinstance(entry, CodecEntity) else entry.json + for entry in cast("tuple[CodecEntity | Opaque, ...]", members[member]) ) return members diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 20ca6cceba..99c3dbf7f8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -15,7 +15,7 @@ DataTypeEntity, Loc, MemberTypes, - MetadataEntity, + Opaque, StorageClass, problem, within, @@ -128,7 +128,7 @@ class StructFieldComponent: """ name: str - data_type: MetadataEntity | object + data_type: DataTypeEntity | Opaque def to_json(self) -> StructField: data_type = self.data_type @@ -137,7 +137,7 @@ def to_json(self) -> StructField: { "name": self.name, "data_type": ( - data_type.to_json() if isinstance(data_type, MetadataEntity) else data_type + data_type.to_json() if isinstance(data_type, DataTypeEntity) else data_type.json ), }, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index c6f3d11790..62f6f5ea84 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -5,15 +5,24 @@ answers for itself. This module is the public door to that layer, for two kinds of caller. -**Reading metadata.** The concrete entities know things a document does -not spell out: what a data type's scalars are, which position a codec -occupies in the pipeline, what a chunk grid divides an array into. Reach -them through a scope: - - from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS - - data_type, problems = CORE_AND_EXTENSIONS.coerce("data_type", "int32") - data_type.storage_class() # 'multi_byte' +**Reading metadata.** `ArrayDocumentV3.from_json` is the fail-fast front +door: one call, and either every extension point is read or a single +`MetadataValidationError` carries every reason it is not. The entities it +yields know things the document does not spell out -- what a data type's +scalars are, which position a codec occupies, what a grid divides an +array into. A name the scope does not model is not a failure: it arrives +as an `Opaque` marked `out_of_scope`, for the reader to resolve +elsewhere. + + from zarr_metadata.v3.entity import ArrayDocumentV3, CodecEntity + + array = ArrayDocumentV3.from_json(json.loads(raw)) # or raises + array.parts.grid.rank + for codec in array.codecs: + if isinstance(codec, CodecEntity): + codec.kind # 'array_bytes' + else: + codec.json, codec.reason # 'out_of_scope': resolve it yourself **Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, `ChunkGridEntity` or `MetadataEntity`, declare `identifier` and @@ -77,6 +86,7 @@ class AcmeLz4Codec(CodecEntity): Loc, MemberTypes, MetadataEntity, + Opaque, StorageClass, TypeCheck, coerce_members, @@ -131,6 +141,7 @@ class AcmeLz4Codec(CodecEntity): "MemberTypes", "MetadataEntity", "NumpyTimeDataType", + "Opaque", "StorageClass", "TypeCheck", "array_problems_v3", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 266bbaf77d..584ebce6bd 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -309,6 +309,7 @@ def test_all_is_grouped_and_unique() -> None: "HexFloat64", "JSONValue", "MetadataValidationError", + "Opaque", "NumpyDatetime64", "NumpyTimeUnit", "NumpyTimedelta64", diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 18c85d127a..b145a0098d 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -11,20 +11,23 @@ import pytest -from zarr_metadata.model import UNSET, ValidationProblem +from zarr_metadata.model import UNSET, MetadataValidationError, ValidationProblem from zarr_metadata.rules import ( canonicalize_array_metadata_v3, validate_array_metadata_v3, ) from zarr_metadata.v3.entity import ( CORE_AND_EXTENSIONS, + ArrayDocumentV3, ArrayParts, + ChunkGridEntity, CodecEntity, CodecKind, Context, DataTypeEntity, MemberTypes, MetadataEntity, + Opaque, StorageClass, is_int, problem, @@ -191,3 +194,64 @@ class Inventive(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.inventive" kind: ClassVar[CodecKind] = "bytes_bytes" member_types: ClassVar[MemberTypes] = {"level": (False, is_int)} + + +def test_a_reader_gets_entities_or_an_exception() -> None: + array = ArrayDocumentV3.from_json(_document()) + assert isinstance(array.data_type, DataTypeEntity) + assert array.data_type.storage_class() == "single_byte" + assert array.parts.grid.rank == 1 + assert [type(codec).identifier for codec in array.codecs if isinstance(codec, CodecEntity)] == [ + "bytes" + ] + + +def test_error_a_reader_gets_every_reason_at_once() -> None: + document = _document(fill_value=-1, dimension_names=("x", "y")) + with pytest.raises(MetadataValidationError) as raised: + ArrayDocumentV3.from_json(document) + assert {problem.loc for problem in raised.value.problems} == { + ("fill_value",), + ("dimension_names",), + } + + +def test_an_unmodelled_extension_is_read_not_refused() -> None: + # Openness: a name this reader does not model is not a failure. It + # arrives as `Opaque`, saying which kind of not-an-entity it is, so + # the reader can resolve it elsewhere instead of guessing. + document = _document( + codecs=( + {"name": "numcodecs.bitround", "configuration": {"keepbits": 9}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ) + ) + array = ArrayDocumentV3.from_json(document) + first = array.codecs[0] + assert isinstance(first, Opaque) + assert first.reason == "out_of_scope" + assert first.json == {"name": "numcodecs.bitround", "configuration": {"keepbits": 9}} + assert isinstance(array.codecs[1], CodecEntity) + + +def test_every_extension_point_is_an_exhaustive_two_case_union() -> None: + # The property that makes the fields narrowable: an entity of the + # right kind, or an `Opaque`. Never a bare `object`. + array = ArrayDocumentV3.from_json(_document(data_type="mycorp.decimal", fill_value=0)) + assert isinstance(array.data_type, (DataTypeEntity, Opaque)) + assert isinstance(array.chunk_grid, (ChunkGridEntity, Opaque)) + assert isinstance(array.chunk_key_encoding, (MetadataEntity, Opaque)) + assert all(isinstance(codec, (CodecEntity, Opaque)) for codec in array.codecs) + + +def test_a_reader_can_choose_its_own_scope() -> None: + document = _document( + codecs=( + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "acme.lz4", "configuration": {"acceleration": 4}}, + ) + ) + assert isinstance(ArrayDocumentV3.from_json(document).codecs[1], Opaque) + in_scope = ArrayDocumentV3.from_json(document, context=SCOPE).codecs[1] + assert isinstance(in_scope, AcmeLz4Codec) + assert in_scope.acceleration == 4 From 61fa11eedc82ca81e4ec9510474b5317ccfdafad Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 19:40:34 +0200 Subject: [PATCH 044/107] refactor(zarr-metadata): serializing is not canonicalizing `to_json` was documented as "the simplest equivalent spelling" and two entities disagreed about whether they meant it: `r008` came back as `r008`, while `[32, 32, 32]` came back run-length encoded. So reading a document and writing it rewrote it -- the same complaint the review made about the model layer, reproduced here, on the `from_json` just added as the reader's front door. They are different operations and now have different methods. `to_json` is faithful to every member the entity holds. `canonical()` returns the simplest equivalent, and `canonicalize_array_metadata_v3` is the only caller -- canonicalizing is something you ask for, not something serialization does to you on the way past. Rectilinear's run-length encoding and blosc's ignored `typesize` move there, and the three entities that contain others canonicalize what they contain. The envelope's spelling is still normalized, because the entity does not model it: a bare name, `{"name": x}` and `{"name": x, "configuration": {}}` all read to the same entity, so all three write back as the bare name. Worth knowing, because zarrs deliberately keeps the object form for readers older than Zarr 3.1. `RectilinearChunkGrid.configuration` is gone with it. It called `super()` to build a member it then discarded, which is what prompted the question. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.8.md | 9 ++ .../src/zarr_metadata/rules/_canonical.py | 6 +- .../src/zarr_metadata/v3/_entity.py | 50 +++++-- .../v3/chunk_grid/rectilinear.py | 10 +- .../src/zarr_metadata/v3/codec/blosc.py | 18 ++- .../src/zarr_metadata/v3/codec/cast_value.py | 6 + .../v3/codec/sharding_indexed.py | 15 +++ .../src/zarr_metadata/v3/data_type/struct.py | 12 ++ .../zarr-metadata/tests/v3/test_entities.py | 123 +++++++++++++++++- 9 files changed, 214 insertions(+), 35 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.8.md b/packages/zarr-metadata/changes/4379.feature.8.md index cac06d2431..3e0cee0d27 100644 --- a/packages/zarr-metadata/changes/4379.feature.8.md +++ b/packages/zarr-metadata/changes/4379.feature.8.md @@ -1,3 +1,12 @@ +Serialization and canonicalization are separate operations. +`entity.to_json()` writes back what was read, member for member, so a +reader that reads a document and writes it returns the bytes it was +given. `entity.canonical()` returns the simplest equivalent, and +`canonicalize_array_metadata_v3` is what asks for it. (The envelope's +spelling is the one thing `to_json` does not preserve, because the entity +does not model it: a bare name, `{"name": x}` and +`{"name": x, "configuration": {}}` all read to the same entity.) + Added `canonicalize_array_metadata_v3`, which answers with `Canonical[T] | Invalid`: a semantically valid document in its simplest equivalent spelling, or every reason it is not valid. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py index 3f0870e9ff..10dc200c0e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py @@ -8,7 +8,7 @@ field narrows to one or the other. Canonical means the simplest spelling with the same meaning, and each -entity decides that for itself in its own `to_json`: an entity whose +entity decides that for itself in its own `canonical`: an entity whose configuration carries nothing collapses to its bare name, `blosc` drops a `typesize` that `shuffle` renders ignored, a rectilinear dimension's chunk sizes run-length encode. This module only collects the answers, and the @@ -69,10 +69,10 @@ def _canonical_document(document: Mapping[str, object], context: Context) -> dic for key in ("data_type", "chunk_grid", "chunk_key_encoding"): entity = getattr(array, key) if isinstance(entity, MetadataEntity): - out[key] = entity.to_json() + out[key] = entity.canonical().to_json() if "codecs" in out: out["codecs"] = tuple( - codec.to_json() if isinstance(codec, MetadataEntity) else codec.json + codec.canonical().to_json() if isinstance(codec, MetadataEntity) else codec.json for codec in array.codecs ) names = out.get("dimension_names") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 8f0926ca5a..f2b76b5bb6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -422,16 +422,35 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: return None, found return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] + def canonical(self) -> Self: + """This entity in the simplest form that means the same thing. + + A *transformation*, asked for by `canonicalize_array_metadata_v3` + and by nothing else. `to_json` does not apply it, because writing + a document back is not the same as asking for it to be rewritten: + a reader that reads and writes should not change bytes it was not + asked to change. + + Default: entities are already canonical. Override where two + spellings of a member mean the same -- a rectilinear dimension's + run-length encoding, a `typesize` that `noshuffle` ignores -- and + where a contained entity has its own canonical form. + """ + return self + def configuration(self) -> dict[str, object]: - """This entity's configuration, in its simplest equivalent form. + """This entity's configuration, as the document would write it. - Absent optional members are left out, which is what makes the - bare-name spelling reachable. Override to drop a member that - another member renders meaningless. + Faithful to every member the entity holds: `to_json` is + serialization, not canonicalization, so nothing is simplified + here. Override only to render a member that is not already JSON, + such as a contained entity. - Absence is `UNSET`, never `None`: this package holds `None` to - mean a JSON `null` the document actually wrote, and `scale_offset` - is a real case where `null` and absent are different documents. + Absent optional members are left out, which is what makes the + bare-name spelling reachable. Absence is `UNSET`, never `None`: + this package holds `None` to mean a JSON `null` the document + actually wrote, and `scale_offset` is a real case where `null` + and absent are different documents. """ return { key: value @@ -448,11 +467,18 @@ def problems(self) -> tuple[ValidationProblem, ...]: return () def to_json(self) -> ZarrV3MetadataFieldJSON: - """This entity in its simplest equivalent spelling. - - A name alone when the name says everything, and the object form - otherwise. `must_understand` is omitted when true, because that is - the default and says nothing; an explicit false says something. + """This entity as a document would write it. + + Faithful to every member: read a document, write it back, and the + members come out as they went in. Ask `canonical` first if you + want the simplest equivalent spelling. + + What is *not* preserved is the envelope's spelling, because the + entity does not model it: a bare name, `{"name": x}`, and + `{"name": x, "configuration": {}}` all mean the same and all read + to the same entity, so all three write back as the bare name. + `must_understand` is omitted when true, which is its default; an + explicit false is kept, because that one says something. Subclasses narrow the return type to their own object TypedDict, which is the JSON form this dataclass models. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 596dc69739..e3beab4254 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -4,8 +4,8 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ -from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import TypedDict @@ -294,15 +294,13 @@ def grid(self, array_shape: object) -> ChunkGrid: """ return ChunkGrid.derived(tuple(_axis_lengths(spec) for spec in self.chunk_shapes)) - def configuration(self) -> dict[str, object]: + def canonical(self) -> Self: """Run-length encoded, which is the spelling that does not grow. Two dimension specs listing the same extents describe the same grid, and the encoded one stays the same size as the array grows. """ - members = super().configuration() - members["chunk_shapes"] = canonical_chunk_shapes(self.chunk_shapes) - return members + return replace(self, chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) def to_json(self) -> RectilinearChunkGridObject: return cast("RectilinearChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 10367ade91..ff33e859ae 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,7 +5,7 @@ """ from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import TypedDict, Unpack @@ -190,17 +190,15 @@ def from_configuration(cls, **configuration: Unpack[BloscCodecConfiguration]) -> """ return cls(**configuration) - def configuration(self) -> dict[str, object]: - """The simplest spelling of this codec's configuration. + def canonical(self) -> Self: + """Without a `typesize` that `noshuffle` renders meaningless. - `typesize` is dropped under `noshuffle`, where the spec says of it - that "the value is ignored" -- so two documents differing only - there describe the same codec. + The spec says of that case that "the value is ignored", so two + documents differing only there describe the same codec. """ - members = super().configuration() - if self.shuffle == BLOSC_NO_SHUFFLE: - members.pop("typesize", None) - return members + if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: + return self + return replace(self, typesize=UNSET) def to_json(self) -> BloscCodecObject: return cast("BloscCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index a7f9bb3a49..e303adc803 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -220,6 +220,12 @@ def problems(self) -> tuple[ValidationProblem, ...]: return () return within(("data_type",), self.data_type.problems()) + def canonical(self) -> Self: + """The target data type in its own canonical form.""" + if not isinstance(self.data_type, DataTypeEntity): + return self + return replace(self, data_type=self.data_type.canonical()) + def configuration(self) -> dict[str, object]: """The target data type in its canonical spelling.""" members = super().configuration() diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a7ff40c4d5..a233b9bb86 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -133,6 +133,13 @@ def _coerce_pipeline( return tuple(coerced), tuple(problems) +def _canonical_pipeline( + codecs: tuple[CodecEntity | Opaque, ...], +) -> tuple[CodecEntity | Opaque, ...]: + """Each codec canonicalized; one out of scope is left as written.""" + return tuple(codec.canonical() if isinstance(codec, CodecEntity) else codec for codec in codecs) + + @dataclass(frozen=True) class ShardingIndexedCodec(CodecEntity): """The `sharding_indexed` codec, coerced from its metadata. @@ -269,6 +276,14 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation ) return tuple(found) + def canonical(self) -> Self: + """Each codec of each pipeline in its own canonical form.""" + return replace( + self, + codecs=_canonical_pipeline(self.codecs), + index_codecs=_canonical_pipeline(self.index_codecs), + ) + def configuration(self) -> dict[str, object]: """The two pipelines in their canonical spelling, entry by entry.""" members = super().configuration() diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 99c3dbf7f8..571e215e71 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -274,6 +274,18 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP ) return tuple(found) + def canonical(self) -> Self: + """Each field's data type in its own canonical form.""" + return replace( + self, + fields=tuple( + replace(field, data_type=field.data_type.canonical()) + if isinstance(field.data_type, DataTypeEntity) + else field + for field in self.fields + ), + ) + def configuration(self) -> dict[str, object]: """Each field in its canonical spelling, type included.""" return {"fields": tuple(field.to_json() for field in self.fields)} diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index c3588f33e6..62ac9a5dc1 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -10,7 +10,7 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING, get_type_hints +from typing import get_type_hints import pytest @@ -67,9 +67,7 @@ from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType - -if TYPE_CHECKING: - from zarr_metadata.v3._entity import MetadataEntity +from zarr_metadata.v3.entity import MetadataEntity # Each registered entity, paired with the TypedDict its constructor # mirrors. Keyed by `:`, because an identifier is only @@ -358,3 +356,120 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: } problems = validate_array_metadata_v3(document) # type: ignore[arg-type] assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "shuffle")] + + +# Entities whose written form and canonical form differ, or could. +FAITHFUL: dict[str, tuple[str, object]] = { + "rectilinear-expanded": ( + "chunk_grid", + { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((32, 32, 32),)}, + }, + ), + "rectilinear-encoded": ( + "chunk_grid", + { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (((32, 3),),)}, + }, + ), + "blosc-ignored-typesize": ( + "codecs", + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "noshuffle", + "blocksize": 0, + "typesize": 4, + }, + }, + ), + "raw-bytes-padded": ("data_type", "r008"), + "scale-offset-explicit-null": ( + "codecs", + {"name": "scale_offset", "configuration": {"offset": None}}, + ), + "must-understand-false": ("codecs", {"name": "crc32c", "must_understand": False}), + "struct-nested": ( + "data_type", + { + "name": "struct", + "configuration": { + "fields": ({"name": "a", "data_type": "uint8"},), + }, + }, + ), +} + + +@pytest.mark.parametrize(("field", "written"), FAITHFUL.values(), ids=list(FAITHFUL)) +def test_to_json_writes_back_what_was_read(field: str, written: object) -> None: + # Serialization is not canonicalization. A reader that reads a + # document and writes it back must not change bytes it was not asked + # to change -- `canonical()` is where you ask. + entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) # type: ignore[arg-type] + assert problems == () + assert isinstance(entity, MetadataEntity) + assert entity.to_json() == written + + +def test_canonical_is_what_simplifies() -> None: + encoded, _ = CORE_AND_EXTENSIONS.coerce( + "chunk_grid", + { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((32, 32, 32),)}, + }, + ) + assert isinstance(encoded, MetadataEntity) + assert encoded.canonical().to_json() == { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (((32, 3),),)}, + } + blosc, _ = CORE_AND_EXTENSIONS.coerce( + "codecs", + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "noshuffle", + "blocksize": 0, + "typesize": 4, + }, + }, + ) + assert isinstance(blosc, MetadataEntity) + assert "typesize" not in blosc.canonical().to_json()["configuration"] # type: ignore[index] + + +def test_canonical_reaches_a_contained_entity() -> None: + shard, _ = CORE_AND_EXTENSIONS.coerce( + "codecs", + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4,), + "codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "noshuffle", + "blocksize": 0, + "typesize": 4, + }, + }, + ), + "index_codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + }, + }, + ) + assert isinstance(shard, MetadataEntity) + inner = shard.canonical().to_json()["configuration"]["codecs"][1] # type: ignore[index] + assert "typesize" not in inner["configuration"] # type: ignore[index] From bfa109cb9ff4d432474de21a190287d98694209d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 21:37:50 +0200 Subject: [PATCH 045/107] refactor(zarr-metadata): an entity cannot exist with values the spec forbids `problems(self)` needing only `self` was the tell: everything it checked was decidable at construction, and deferring it meant an invalid entity could exist. It no longer can. `value_problems` is a routine over the members, `unchecked` builds without asking, and the dataclass constructor asks then builds. The justification for the old arrangement -- that a value-invalid entity still answers the composition checks -- did not survive scrutiny. What those checks read across entities is `kind`, `variable_size`, `identifier`: class variables, which a failed construction still has. What is genuinely lost is the failed entity's own instance data, and losing that is right: you have to fix it first. `must_understand` becomes a class variable and leaves the configuration. It belongs to the kind of metadata, not to a use of it, so the spec's carve-out permitting `false` on a codec is refused here -- while the case where the flag means something, an unknown top-level extension field, keeps it per-occurrence on `ZarrV3NamedConfig`. Two seams the refactor needed. `prepare` reads members that are themselves entities before values are judged, so `struct` can ask whether a field is fixed-size. And the container entities get a TypedDict for their members *as held* -- `StructMembers`, `ShardingIndexedMembers` -- because after `prepare` those are entities, not the JSON their configuration TypedDict describes. Writing that down beats casting over it. Measured over 800 documents against the rule registry: 89 report fewer problems, none changes verdict. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.10.md | 30 ++++ .../src/zarr_metadata/model/_validation.py | 19 +- .../src/zarr_metadata/v3/_document.py | 19 +- .../src/zarr_metadata/v3/_entity.py | 164 ++++++++++++++---- .../v3/chunk_grid/rectilinear.py | 78 +++++---- .../zarr_metadata/v3/chunk_grid/regular.py | 42 +++-- .../src/zarr_metadata/v3/codec/blosc.py | 90 +++++----- .../src/zarr_metadata/v3/codec/cast_value.py | 20 +-- .../src/zarr_metadata/v3/codec/gzip.py | 21 ++- .../zarr_metadata/v3/codec/scale_offset.py | 39 +++-- .../v3/codec/sharding_indexed.py | 84 +++++---- .../src/zarr_metadata/v3/codec/transpose.py | 35 ++-- .../src/zarr_metadata/v3/codec/zstd.py | 27 +-- .../v3/data_type/numpy_datetime64.py | 34 ++-- .../v3/data_type/numpy_timedelta64.py | 34 ++-- .../src/zarr_metadata/v3/data_type/raw.py | 46 +++-- .../src/zarr_metadata/v3/data_type/struct.py | 112 ++++++------ .../zarr-metadata/tests/model/test_array.py | 14 +- .../tests/rules/test_canonical.py | 13 +- .../tests/rules/test_chunk_grid.py | 28 +-- .../tests/rules/test_v3_array_rules.py | 10 +- .../zarr-metadata/tests/v3/test_entities.py | 28 ++- .../tests/v3/test_extension_api.py | 24 ++- .../tests/v3/test_fill_values.py | 12 +- 24 files changed, 632 insertions(+), 391 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.feature.10.md diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md new file mode 100644 index 0000000000..66768fbedc --- /dev/null +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -0,0 +1,30 @@ +An entity is the value guarantee, not just the type one. Construction +refuses values the spec disallows, so `BloscCodec(clevel=99)` raises +rather than serializing a document no reader will accept, and there is no +such thing as an invalid entity to hold. Code that wants unvalidated +metadata already has somewhere to put it: `Opaque`, which carries the +JSON verbatim. + +Three pieces, as the structure requires. `value_problems` is a routine +over the members rather than a method on an entity, because judging +values does not need one -- and needing one would mean an invalid one had +been built. `unchecked` builds without asking, for a caller that has +already asked. The dataclass constructor asks, then builds. + +`must_understand` moves to a class variable and out of the configuration +entirely. It is a property of the *kind* of metadata -- a codec is +something you must understand every time it appears, consolidated +metadata is unconditionally skippable -- not a per-occurrence choice. The +spec permits `must_understand: false` on a codec; this package reads that +as an oversight and refuses it at every extension point, while leaving it +where it earns its keep: an unknown top-level extension field a reader +really can skip. + +The cost is reporting density on documents that are already invalid. An +entity that cannot be built cannot be asked anything, so a grid with one +bad extent no longer pins its good axes and a shard with one bad inner +extent no longer judges its pipelines. Measured against the rule registry +this layer replaced, over 800 documents: 89 report fewer problems than +before, and **no document changes verdict**. The report that survives is +the one that has to be fixed first, and the JSON is still there on the +`Opaque` standing in for the entity. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 3ba683136e..27becad824 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -522,6 +522,13 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: problems.extend(_validate_dim_sequence(doc, "shape")) if "fill_value" in doc: problems.extend(_prefix("fill_value", validate_json(doc["fill_value"]))) + # Every extension *point* must be understood: ignoring a codec gives + # wrong bytes just as surely as ignoring a data type gives wrong + # values. The spec names only the first three + # (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), + # which this package reads as an oversight rather than a licence. + # `must_understand: false` keeps its meaning where it has one: an + # unknown top-level extension *field*, which a reader really can skip. for key in ("data_type", "chunk_grid", "chunk_key_encoding"): if key in doc: problems.extend( @@ -543,7 +550,17 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: ) ) for index, entry in enumerate(cast("Sequence[object]", entries)): - problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry)))) + problems.extend( + _prefix( + key, + _prefix( + index, + validate_metadata_field_v3( + entry, allow_must_understand_false=False + ), + ), + ) + ) if "attributes" in doc: problems.extend(_validate_attributes(doc["attributes"])) if "dimension_names" in doc: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 5632639838..12cda0ea8f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -74,8 +74,9 @@ def problems(self) -> tuple[ValidationProblem, ...]: The type-space problems are `read_array_v3`'s, because they are the reasons some of this is `Opaque` rather than an entity. """ + # No per-entity value problems: an entity exists only if its own + # values are allowed, so `read_array_v3` has already reported any. return ( - *_entity_problems(self), *_fill_value_problems(self), *_grid_problems(self), *_dimension_names_problems(self), @@ -185,22 +186,6 @@ def read_array_v3( ) -def _entity_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: - """What each entity says is wrong with its own values.""" - found: list[ValidationProblem] = [] - for _, key in _SINGLE_FIELDS: - entity = getattr(array, key) - if isinstance(entity, MetadataEntity): - found.extend(within((key,), entity.problems())) - for _, key in _SEQUENCE_FIELDS: - for index, entity in enumerate( - cast("tuple[MetadataEntity | Opaque, ...]", getattr(array, key)) - ): - if isinstance(entity, MetadataEntity): - found.extend(within((key, index), entity.problems())) - return tuple(found) - - def _fill_value_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: """The fill value, judged by the data type it fills.""" if not isinstance(array.data_type, DataTypeEntity) or "fill_value" not in array.document: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index f2b76b5bb6..8178342f48 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -39,14 +39,18 @@ from __future__ import annotations from collections.abc import Mapping as _Mapping -from dataclasses import dataclass, field +from dataclasses import MISSING, dataclass, fields from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast from typing_extensions import TypeIs from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem, is_json +from zarr_metadata.model._validation import ( + MetadataValidationError, + ValidationProblem, + is_json, +) from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: @@ -254,6 +258,15 @@ def coerce_members( return members, tuple(problems), frozenset(unreadable) +ValueRoutine: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" +"""An entity's value-space judgment, over the members it was given.""" + + +def _no_value_problems(**members: object) -> tuple[ValidationProblem, ...]: + """An entity whose types admit only valid values has nothing to add.""" + return () + + @dataclass(frozen=True, slots=True) class Opaque: """A metadata field this reading did not turn into an entity. @@ -305,11 +318,21 @@ class MetadataEntity: rather than a constant, a member another member renders meaningless. """ - # Keyword-only: it is the envelope's member, not the configuration's, - # and it would otherwise take the first positional slot of every - # entity -- so `RawBytesDataType("r16")` would set this instead of - # the field it reads as. - must_understand: bool = field(default=True, kw_only=True) + must_understand: ClassVar[bool] = True + """Whether a reader that does not know this entity may skip it. + + A property of the *kind* of metadata, not of a use of it: a codec is + something you must understand, every time it appears, because + ignoring one gives wrong bytes. Consolidated metadata is the opposite + and is unconditionally skippable. Neither is a per-occurrence choice, + so neither is a configuration member -- which is why this is a class + variable and not a field. + + The spec permits `must_understand: false` on a codec; this package + treats that as an oversight and refuses it. Where the flag does earn + its keep -- an unknown top-level extension field a reader really can + skip -- it stays per-occurrence, on `ZarrV3NamedConfig`. + """ identifier: ClassVar[str] """The name this entity is registered under. @@ -348,6 +371,15 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: super().__init_subclass__(**kwargs) if base: return + if "problems" in cls.__dict__: + # Value rules are `value_problems`, a static routine over the + # members. An override named `problems` is a rule that would + # never run, and nothing else would say so. + msg = ( + f"{cls.__name__} defines `problems`; value rules belong in " + "`value_problems`, which takes the members rather than an entity" + ) + raise TypeError(msg) missing = [name for name in cls.required_class_vars if not hasattr(cls, name)] if len(missing) != 0: msg = f"{cls.__name__} does not declare {', '.join(missing)}" @@ -379,6 +411,19 @@ def accepts(cls, name: str) -> bool: """ return name == cls.identifier + @classmethod + def prepare( + cls, members: dict[str, object], context: Context + ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """The members, with any that are themselves entities read as such. + + The seam between `coerce_members`, which knows types, and + `value_problems`, which knows values: a `struct` cannot ask + whether a field is fixed-size until that field's data type is an + entity. Default: nothing to convert. + """ + return members, () + @classmethod def coerce(cls, value: object, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one. @@ -386,7 +431,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: `context` is the scope this reading is happening in; most entities have no use for it and ignore it. """ - name, configuration, must_understand = named_configuration(value) + name, configuration, _ = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), f"expected the {cls.identifier!r} entity") if configuration is None: @@ -398,29 +443,22 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: ) configuration = cast("Mapping[str, object]", {}) members, found, unreadable = coerce_members(configuration, cls.member_types) + if len(unreadable) == 0: + # Before judging: a member that is itself an entity has to be + # one before its container's value rules can ask it anything. + members, nested = cls.prepare(members, context) + found = (*found, *nested) if len(unreadable) != 0: - # The entity cannot be built, but the members that *did* read - # can still be judged -- one bad member should not hide the - # value problems of the ones beside it. Anything the partial - # reading says about an unreadable member is its default - # talking, so those are dropped. - partial = cls(must_understand=must_understand, **members) # type: ignore[arg-type] - found = ( - *found, - # `within`, because a partial reading reports relative to - # the configuration and `coerce`'s caller does not insert - # that segment -- `coerce_members` problems already carry it. - *within( - (), - [ - entry - for entry in partial.problems() - if entry.loc[:1] not in {(key,) for key in unreadable} - ], - ), - ) + # A member that could not be read leaves a hole, and the value + # rules are written over a whole configuration -- blosc's + # `typesize` requirement reads `shuffle`. Judging around the + # hole would be guessing, so the type problems stand alone. return None, found - return cls(must_understand=must_understand, **members), found # type: ignore[arg-type] + found = (*found, *within((), cls.value_problems(**members))) # type: ignore[arg-type] + if any(entry.kind != "unknown_key" for entry in found): + return None, found + # Already asked, so do not ask again on the way in. + return cls.unchecked(**members), found def canonical(self) -> Self: """This entity in the simplest form that means the same thing. @@ -452,19 +490,68 @@ def configuration(self) -> dict[str, object]: actually wrote, and `scale_offset` is a real case where `null` and absent are different documents. """ + return self._members() + + value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) + """Every value among the members the spec disallows. + + A routine rather than a method, because judging values does not need + an entity -- and needing one would mean an invalid one had been + built. Each entity supplies its own, taking + `Unpack[Configuration]`: the same spelling the constructor + takes, receiving only the members that are present. + + Typed loosely here because the base does not know any entity's + configuration, and saying so is the truth. Call a specific routine by + its own name to have the arguments checked. + + Locations are relative to the entity's `configuration`. + """ + + def __post_init__(self) -> None: + """Refuse to exist with values the spec disallows. + + So an instance is the value guarantee, not just the type one: + `BloscCodec(clevel=99)` raises rather than serializing a document + no reader will accept. `coerce` asks `value_problems` first and + reports, so reading a bad document still returns problems rather + than raising, and `unchecked` is the door for a caller that has + already asked. + """ + found = type(self).value_problems(**self._members()) + if len(found) != 0: + raise MetadataValidationError(found) + + def _members(self) -> dict[str, object]: + """The configuration members present on this entity, unrendered. + + What `value_problems` and `configuration` are both built from; + `configuration` may render a member that is itself an entity, and + `value_problems` wants it as it is. + """ return { key: value for key in type(self).member_types if (value := getattr(self, key)) is not UNSET } - def problems(self) -> tuple[ValidationProblem, ...]: - """Every value of this entity the spec disallows. + @classmethod + def unchecked(cls, **members: object) -> Self: + """This entity, without asking whether its values are allowed. - Locations are relative to the entity's `configuration`. Default: - an entity whose type admits only valid values has nothing to add. + For a caller that has already asked -- `coerce` does, so that it + can report the answer instead of raising it. Named so that + choosing it is deliberate. """ - return () + entity = object.__new__(cls) + for field_ in fields(cls): + if field_.name in members: + object.__setattr__(entity, field_.name, members[field_.name]) + elif field_.default is not MISSING: + object.__setattr__(entity, field_.name, field_.default) + elif field_.default_factory is not MISSING: # pragma: no cover - none today + object.__setattr__(entity, field_.name, field_.default_factory()) + return entity def to_json(self) -> ZarrV3MetadataFieldJSON: """This entity as a document would write it. @@ -477,19 +564,19 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: entity does not model it: a bare name, `{"name": x}`, and `{"name": x, "configuration": {}}` all mean the same and all read to the same entity, so all three write back as the bare name. - `must_understand` is omitted when true, which is its default; an - explicit false is kept, because that one says something. + `must_understand` follows the entity's own class variable, so it + is omitted for everything this package models today. Subclasses narrow the return type to their own object TypedDict, which is the JSON form this dataclass models. """ configuration = self.configuration() - if len(configuration) == 0 and self.must_understand: + if len(configuration) == 0 and type(self).must_understand: return cast("ZarrV3MetadataFieldJSON", type(self).identifier) entry: dict[str, object] = {"name": type(self).identifier} if len(configuration) != 0: entry["configuration"] = configuration - if not self.must_understand: + if not type(self).must_understand: entry["must_understand"] = False return cast("ZarrV3MetadataFieldJSON", entry) @@ -650,6 +737,7 @@ def named_configuration( "Opaque", "StorageClass", "TypeCheck", + "ValueRoutine", "coerce_members", "is_bool", "is_int", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index e3beab4254..d2551b2aed 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -7,13 +7,14 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, Loc, MemberTypes, + ValueRoutine, is_integer, one_of, problem, @@ -201,6 +202,44 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: return frozenset(lengths) if len(lengths) != 0 else None +def _value_problems( + **members: Unpack[RectilinearChunkGridConfiguration], +) -> tuple[ValidationProblem, ...]: + """Every chunk extent, bare or run-length encoded, must be positive. + + A run's count must be positive too: a run of zero chunks is a way + of writing nothing at all, and the empty spelling already exists. + """ + found: list[ValidationProblem] = [] + for dim, spec in enumerate(members["chunk_shapes"]): + loc: tuple[str | int, ...] = ("chunk_shapes", dim) + if isinstance(spec, int): + if spec < 1: + found.extend( + problem(loc, f"expected a positive chunk extent, got {spec}", "invalid_value") + ) + continue + for position, item in enumerate(spec): + if isinstance(item, int): + if item < 1: + found.extend( + problem( + (*loc, position), + f"expected a positive chunk extent, got {item}", + "invalid_value", + ) + ) + elif item[0] < 1 or item[1] < 1: + found.extend( + problem( + (*loc, position), + f"expected a positive [size, count] pair, got {item!r}", + "invalid_value", + ) + ) + return tuple(found) + + @dataclass(frozen=True) class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" @@ -216,42 +255,7 @@ class RectilinearChunkGrid(ChunkGridEntity): "chunk_shapes": (True, _is_dim_specs), } - def problems(self) -> tuple[ValidationProblem, ...]: - """Every chunk extent, bare or run-length encoded, must be positive. - - A run's count must be positive too: a run of zero chunks is a way - of writing nothing at all, and the empty spelling already exists. - """ - found: list[ValidationProblem] = [] - for dim, spec in enumerate(self.chunk_shapes): - loc: tuple[str | int, ...] = ("chunk_shapes", dim) - if isinstance(spec, int): - if spec < 1: - found.extend( - problem( - loc, f"expected a positive chunk extent, got {spec}", "invalid_value" - ) - ) - continue - for position, item in enumerate(spec): - if isinstance(item, int): - if item < 1: - found.extend( - problem( - (*loc, position), - f"expected a positive chunk extent, got {item}", - "invalid_value", - ) - ) - elif item[0] < 1 or item[1] < 1: - found.extend( - problem( - (*loc, position), - f"expected a positive [size, count] pair, got {item!r}", - "invalid_value", - ) - ) - return tuple(found) + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 3525a7d03e..8a9e9571bc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -7,12 +7,13 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, MemberTypes, + ValueRoutine, is_int, problem, sequence_of, @@ -63,6 +64,27 @@ class RegularChunkGridObject(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[RegularChunkGridConfiguration], +) -> tuple[ValidationProblem, ...]: + """Every chunk extent must be at least one element. + + A chunk of zero elements along an axis covers nothing, so no + finite number of them tiles the axis; a negative one is + meaningless. Whether there is one extent *per array dimension* is + a question for the document, and the rules layer asks it. + """ + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(members["chunk_shape"]) + if extent < 1 + ) + + @dataclass(frozen=True) class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" @@ -74,23 +96,7 @@ class RegularChunkGrid(ChunkGridEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"chunk_shape": (True, sequence_of(is_int))} - def problems(self) -> tuple[ValidationProblem, ...]: - """Every chunk extent must be at least one element. - - A chunk of zero elements along an axis covers nothing, so no - finite number of them tiles the axis; a negative one is - meaningless. Whether there is one extent *per array dimension* is - a question for the document, and the rules layer asks it. - """ - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(self.chunk_shape) - if extent < 1 - ) + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index ff33e859ae..e58a16dc9b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -16,6 +16,7 @@ CodecEntity, CodecKind, MemberTypes, + ValueRoutine, is_int, one_of, problem, @@ -108,6 +109,44 @@ def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, ] +def _value_problems( + **members: Unpack[BloscCodecConfiguration], +) -> tuple[ValidationProblem, ...]: + """The value constraints the spec places on a blosc configuration.""" + found: list[ValidationProblem] = [] + clevel = members["clevel"] + if not 0 <= clevel <= 9: + found.extend( + problem(("clevel",), f"expected an integer in [0, 9], got {clevel}", "invalid_value") + ) + blocksize = members["blocksize"] + if blocksize < 0: + found.extend( + problem( + ("blocksize",), + f"expected a non-negative integer, got {blocksize}", + "invalid_value", + ) + ) + shuffle = members["shuffle"] + typesize = members.get("typesize") + # Only where it means something: under `noshuffle` the spec says + # "the value is ignored", and `canonical` drops it. + if typesize is not None and shuffle != BLOSC_NO_SHUFFLE and typesize < 1: + found.extend( + problem(("typesize",), f"expected a positive integer, got {typesize}", "invalid_value") + ) + if shuffle != BLOSC_NO_SHUFFLE and typesize is None: + found.extend( + problem( + ("typesize",), + f"typesize is required when shuffle is {shuffle!r}", + "missing_key", + ) + ) + return tuple(found) + + @dataclass(frozen=True) class BloscCodec(CodecEntity): """The `blosc` codec, coerced from its metadata. @@ -139,56 +178,7 @@ class BloscCodec(CodecEntity): "typesize": (False, is_int), } - def problems(self) -> tuple[ValidationProblem, ...]: - """The value constraints the spec places on a blosc configuration.""" - found: list[ValidationProblem] = [] - if not 0 <= self.clevel <= 9: - found.extend( - problem( - ("clevel",), - f"expected an integer in [0, 9], got {self.clevel}", - "invalid_value", - ) - ) - if self.blocksize < 0: - found.extend( - problem( - ("blocksize",), - f"expected a non-negative integer, got {self.blocksize}", - "invalid_value", - ) - ) - # Only where it means something: under `noshuffle` the spec says - # "the value is ignored" and `configuration` drops it, so judging - # it would let `to_json` turn an invalid codec into a valid - # document. - if self.typesize is not UNSET and self.shuffle != BLOSC_NO_SHUFFLE and self.typesize < 1: - found.extend( - problem( - ("typesize",), - f"expected a positive integer, got {self.typesize}", - "invalid_value", - ) - ) - if self.shuffle != BLOSC_NO_SHUFFLE and self.typesize is UNSET: - found.extend( - problem( - ("typesize",), - f"typesize is required when shuffle is {self.shuffle!r}", - "missing_key", - ) - ) - return tuple(found) - - @classmethod - def from_configuration(cls, **configuration: Unpack[BloscCodecConfiguration]) -> Self: - """This codec from its configuration members. - - The configuration TypedDict unpacked *is* this constructor's - signature, so a caller with a well-typed configuration builds a - well-typed codec, and a type checker says so at the call site. - """ - return cls(**configuration) + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def canonical(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index e303adc803..d42c90a6a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -14,7 +14,6 @@ DATA_TYPE, CodecEntity, CodecKind, - Coerced, DataTypeEntity, Loc, MemberTypes, @@ -22,7 +21,6 @@ is_json_value, one_of, problem, - within, ) from zarr_metadata.v3._parts import ArrayParts @@ -205,20 +203,14 @@ class CastValueCodec(CodecEntity): } @classmethod - def coerce(cls, value: object, context: "Context") -> Coerced[Self]: - codec, problems = super().coerce(value, context) - if codec is None: - return None, problems + def prepare( + cls, members: dict[str, object], context: "Context" + ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """The target data type, read in this scope.""" data_type, found = context.coerce( - DATA_TYPE, codec.data_type, ("configuration", "data_type") + DATA_TYPE, members["data_type"], ("configuration", "data_type") ) - return replace(codec, data_type=data_type), (*problems, *found) - - def problems(self) -> tuple[ValidationProblem, ...]: - """Whatever the data type being cast to says about itself.""" - if not isinstance(self.data_type, DataTypeEntity): - return () - return within(("data_type",), self.data_type.problems()) + return {**members, "data_type": data_type}, found def canonical(self) -> Self: """The target data type in its own canonical form.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 0bee831002..61404f3f35 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -7,13 +7,14 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, MemberTypes, + ValueRoutine, is_int, problem, ) @@ -68,6 +69,16 @@ class GzipCodecObject(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[GzipCodecConfiguration], +) -> tuple[ValidationProblem, ...]: + """gzip compression levels run 0 to 9.""" + level = members["level"] + if not 0 <= level <= 9: + return problem(("level",), f"expected an integer in [0, 9], got {level}", "invalid_value") + return () + + @dataclass(frozen=True) class GzipCodec(CodecEntity): """The `gzip` codec, coerced from its metadata.""" @@ -81,13 +92,7 @@ class GzipCodec(CodecEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"level": (True, is_int)} - def problems(self) -> tuple[ValidationProblem, ...]: - """gzip compression levels run 0 to 9.""" - if not 0 <= self.level <= 9: - return problem( - ("level",), f"expected an integer in [0, 9], got {self.level}", "invalid_value" - ) - return () + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def to_json(self) -> GzipCodecObject: return cast("GzipCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index c5f7087ee0..7a614f3c8d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET @@ -16,6 +16,7 @@ CodecEntity, CodecKind, MemberTypes, + ValueRoutine, is_json_value, problem, ) @@ -75,6 +76,27 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[ScaleOffsetCodecConfiguration], +) -> tuple[ValidationProblem, ...]: + """Each value is a scalar of the array's type, so neither is null. + + The registry says each is "JSON-encoded per the input array's + fill-value rules", and no data type admits `null` as a fill value. + Which scalar it should be needs the data type, so that part is the + document's question, not this codec's. + """ + # Each member named outright: a TypedDict indexed by a loop variable + # has no type, and the two are different members rather than two of + # a kind. + found: list[ValidationProblem] = [] + if members.get("offset", UNSET) is None: + found.extend(problem(("offset",), "expected a scalar, got null", "invalid_value")) + if members.get("scale", UNSET) is None: + found.extend(problem(("scale",), "expected a scalar, got null", "invalid_value")) + return tuple(found) + + @dataclass(frozen=True) class ScaleOffsetCodec(CodecEntity): """The `scale_offset` codec, coerced from its metadata. @@ -103,20 +125,7 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: """ return incoming - def problems(self) -> tuple[ValidationProblem, ...]: - """Each value is a scalar of the array's type, so neither is null. - - The registry says each is "JSON-encoded per the input array's - fill-value rules", and no data type admits `null` as a fill value. - Which scalar it should be needs the data type, so that part is the - document's question, not this codec's. - """ - return tuple( - found - for member in ("offset", "scale") - if getattr(self, member) is None - for found in problem((member,), "expected a scalar, got null", "invalid_value") - ) + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a233b9bb86..cdd3ab034a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -15,15 +15,14 @@ CODECS, CodecEntity, CodecKind, - Coerced, Loc, MemberTypes, Opaque, + ValueRoutine, is_int, one_of, problem, sequence_of, - within, ) from zarr_metadata.v3._parts import ( UNKNOWN_GRID, @@ -36,7 +35,7 @@ if TYPE_CHECKING: from zarr_metadata.v3._registry import Context -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON @@ -140,6 +139,39 @@ def _canonical_pipeline( return tuple(codec.canonical() if isinstance(codec, CodecEntity) else codec for codec in codecs) +class ShardingIndexedMembers(TypedDict): + """A shard's members as the entity holds them. + + Not `ShardingIndexedCodecConfiguration`, which describes the JSON: by + the time values are judged, `prepare` has read the two pipelines, so + these are codecs rather than the metadata fields that named them. + """ + + chunk_shape: tuple[int, ...] + codecs: tuple[CodecEntity | Opaque, ...] + index_codecs: tuple[CodecEntity | Opaque, ...] + index_location: NotRequired[ShardingIndexLocation] + + +def _value_problems( + **members: Unpack[ShardingIndexedMembers], +) -> tuple[ValidationProblem, ...]: + """Every inner chunk extent must be at least one element. + + Nothing about the two pipelines: their codecs are entities, and an + entity exists only if its own values are allowed. + """ + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(members["chunk_shape"]) + if extent < 1 + ) + + @dataclass(frozen=True) class ShardingIndexedCodec(CodecEntity): """The `sharding_indexed` codec, coerced from its metadata. @@ -166,44 +198,26 @@ class ShardingIndexedCodec(CodecEntity): "index_location": (False, one_of(SHARDING_INDEX_LOCATION)), } + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @classmethod - def coerce(cls, value: object, context: "Context") -> Coerced[Self]: - shard, problems = super().coerce(value, context) - if shard is None: - return None, problems - inner, from_inner = _coerce_pipeline(shard.codecs, context, ("configuration", "codecs")) + def prepare( + cls, members: dict[str, object], context: "Context" + ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """Both pipelines, read in this scope.""" + inner, from_inner = _coerce_pipeline( + cast("tuple[object, ...]", members["codecs"]), context, ("configuration", "codecs") + ) index, from_index = _coerce_pipeline( - shard.index_codecs, context, ("configuration", "index_codecs") + cast("tuple[object, ...]", members["index_codecs"]), + context, + ("configuration", "index_codecs"), ) return ( - replace(shard, codecs=inner, index_codecs=index), - (*problems, *from_inner, *from_index), + {**members, "codecs": inner, "index_codecs": index}, + (*from_inner, *from_index), ) - def problems(self) -> tuple[ValidationProblem, ...]: - """This shard's own values, and those of the codecs it holds. - - Whether the two pipelines are well *formed* -- one array-to-bytes - codec, in the right order -- spans the whole chain, so the rules - layer asks that. - """ - found: list[ValidationProblem] = [ - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(self.chunk_shape) - if extent < 1 - ] - for member in ("codecs", "index_codecs"): - for position, codec in enumerate( - cast("tuple[CodecEntity | Opaque, ...]", getattr(self, member)) - ): - if isinstance(codec, CodecEntity): - found.extend(within((member, position), codec.problems())) - return tuple(found) - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """This shard against the array reaching it, and its two pipelines. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 7fd016b9d8..676a106f6b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -7,13 +7,14 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, MemberTypes, + ValueRoutine, is_int, problem, sequence_of, @@ -65,6 +66,24 @@ class TransposeCodecObject(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[TransposeCodecConfiguration], +) -> tuple[ValidationProblem, ...]: + """`order` must permute its own axes. + + Whether it permutes the *array's* axes is a different question -- + it needs the array's rank -- and the rules layer asks that one. + """ + order = members["order"] + if sorted(order) != list(range(len(order))): + return problem( + ("order",), + f"expected a permutation of 0..{len(order) - 1}, got {order!r}", + "invalid_value", + ) + return () + + @dataclass(frozen=True) class TransposeCodec(CodecEntity): """The `transpose` codec, coerced from its metadata.""" @@ -77,19 +96,7 @@ class TransposeCodec(CodecEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"order": (True, sequence_of(is_int))} - def problems(self) -> tuple[ValidationProblem, ...]: - """`order` must permute its own axes. - - Whether it permutes the *array's* axes is a different question -- - it needs the array's rank -- and the rules layer asks that one. - """ - if sorted(self.order) != list(range(len(self.order))): - return problem( - ("order",), - f"expected a permutation of 0..{len(self.order) - 1}, got {self.order!r}", - "invalid_value", - ) - return () + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 772633b037..b163f2abfb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Unpack from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem @@ -17,6 +17,7 @@ CodecEntity, CodecKind, MemberTypes, + ValueRoutine, is_bool, is_int, problem, @@ -77,6 +78,20 @@ class ZstdCodecObject(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[ZstdCodecConfiguration], +) -> tuple[ValidationProblem, ...]: + """zstd compression levels run -131072 to 22.""" + level = members["level"] + if not ZSTD_MIN_LEVEL <= level <= ZSTD_MAX_LEVEL: + return problem( + ("level",), + f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {level}", + "invalid_value", + ) + return () + + @dataclass(frozen=True) class ZstdCodec(CodecEntity): """The `zstd` codec, coerced from its metadata.""" @@ -94,15 +109,7 @@ class ZstdCodec(CodecEntity): "checksum": (False, is_bool), } - def problems(self) -> tuple[ValidationProblem, ...]: - """zstd compression levels run -131072 to 22.""" - if not ZSTD_MIN_LEVEL <= self.level <= ZSTD_MAX_LEVEL: - return problem( - ("level",), - f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {self.level}", - "invalid_value", - ) - return () + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def to_json(self) -> ZstdCodecObject: return cast("ZstdCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 204fd99c9d..10bed5b53d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -7,12 +7,13 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( MemberTypes, StorageClass, + ValueRoutine, is_int, one_of, problem, @@ -77,6 +78,23 @@ class NumpyDatetime64(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[NumpyDatetime64Configuration], +) -> tuple[ValidationProblem, ...]: + """`scale_factor` counts units per step, so it is positive. + + The upper bound is numpy's: the field is a signed 32-bit integer. + """ + scale_factor = members["scale_factor"] + if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + return problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", + "invalid_value", + ) + return () + + @dataclass(frozen=True) class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" @@ -93,19 +111,7 @@ class NumpyDatetime64DataType(NumpyTimeDataType): "scale_factor": (True, is_int), } - def problems(self) -> tuple[ValidationProblem, ...]: - """`scale_factor` counts units per step, so it is positive. - - The upper bound is numpy's: the field is a signed 32-bit integer. - """ - if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - return problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " - f"got {self.scale_factor}", - "invalid_value", - ) - return () + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def to_json(self) -> NumpyDatetime64: return cast("NumpyDatetime64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 739f576c32..b6a31f2f3a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -7,12 +7,13 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( MemberTypes, StorageClass, + ValueRoutine, is_int, one_of, problem, @@ -97,6 +98,23 @@ class NumpyTimedelta64(TypedDict, closed=True): ] +def _value_problems( + **members: Unpack[NumpyTimedelta64Configuration], +) -> tuple[ValidationProblem, ...]: + """`scale_factor` counts units per step, so it is positive. + + The upper bound is numpy's: the field is a signed 32-bit integer. + """ + scale_factor = members["scale_factor"] + if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + return problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", + "invalid_value", + ) + return () + + @dataclass(frozen=True) class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" @@ -113,19 +131,7 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): "scale_factor": (True, is_int), } - def problems(self) -> tuple[ValidationProblem, ...]: - """`scale_factor` counts units per step, so it is positive. - - The upper bound is numpy's: the field is a signed 32-bit integer. - """ - if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - return problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " - f"got {self.scale_factor}", - "invalid_value", - ) - return () + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) def to_json(self) -> NumpyTimedelta64: return cast("NumpyTimedelta64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 05607462e6..fe583da854 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -12,7 +12,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, NewType, Self, cast -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( Coerced, @@ -84,6 +84,19 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: ] +def _name_problems(name: str) -> tuple[ValidationProblem, ...]: + """Why `name` is not a well-formed `r`, if it is not. + + "raw bits, variable size given by *, limited to be a multiple of 8" + -- and zero bits is not a data type. + """ + try: + raw_bytes_dtype_name(name) + except ValueError as error: + return problem((), str(error), "invalid_value") + return () + + @dataclass(frozen=True) class RawBytesDataType(DataTypeEntity): """An `r` raw-bytes data type, coerced from its metadata. @@ -115,7 +128,7 @@ def accepts(cls, name: str) -> bool: @classmethod def coerce(cls, value: object, context: object) -> Coerced[Self]: - name, configuration, must_understand = named_configuration(value) + name, configuration, _ = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), "expected an 'r' raw-bytes data type") found: tuple[ValidationProblem, ...] = () @@ -125,27 +138,24 @@ def coerce(cls, value: object, context: object) -> Coerced[Self]: # its fill values are still judged. Returning nothing here let # a stray key hide every other problem in the document. found = problem(("configuration",), "'r' takes no configuration", "unknown_key") - return cls(must_understand=must_understand, data_type_name=name), found + found = (*found, *_name_problems(name)) + if any(entry.kind != "unknown_key" for entry in found): + return None, found + return cls.unchecked(data_type_name=name), found - def problems(self) -> tuple[ValidationProblem, ...]: - """N must be a positive multiple of 8. + def __post_init__(self) -> None: + """This family's validity is in its name, not a configuration. - "raw bits, variable size given by *, limited to be a multiple of - 8" -- and zero bits is not a data type. + So the base's member-driven check has nothing to look at, and + this one supplies it. """ - try: - raw_bytes_dtype_name(self.data_type_name) - except ValueError as error: - return problem((), str(error), "invalid_value") - return () + super().__post_init__() + found = _name_problems(self.data_type_name) + if len(found) != 0: + raise MetadataValidationError(found) def to_json(self) -> ZarrV3MetadataFieldJSON: - if self.must_understand: - return cast("ZarrV3MetadataFieldJSON", self.data_type_name) - return cast( - "ZarrV3MetadataFieldJSON", - {"name": self.data_type_name, "must_understand": False}, - ) + return cast("ZarrV3MetadataFieldJSON", self.data_type_name) def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """One byte value per byte of the scalar. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 571e215e71..65db5f252c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -11,20 +11,19 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( DATA_TYPE, - Coerced, DataTypeEntity, Loc, MemberTypes, Opaque, StorageClass, + ValueRoutine, problem, - within, ) if TYPE_CHECKING: from zarr_metadata.v3._registry import Context -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, Unpack from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON @@ -143,6 +142,58 @@ def to_json(self) -> StructField: ) +class StructMembers(TypedDict): + """A struct's members as the entity holds them. + + Not `StructConfiguration`, which describes the JSON: by the time + values are judged, `prepare` has read each field's data type, so + these are components holding entities rather than field objects. + """ + + fields: tuple[StructFieldComponent, ...] + + +def _value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ...]: + """What a struct can judge about its own fields. + + Names have to exist, be non-empty and be distinct, because a fill + value addresses fields by name. Field types have to be fixed-size, + because a record's layout is otherwise not determined. Nothing about + a field type's own values: it is an entity, so it exists only if + those are allowed. + """ + fields = members["fields"] + found: list[ValidationProblem] = [] + if len(fields) == 0: + found.extend(problem(("fields",), "expected at least one struct field", "invalid_value")) + seen: dict[str, int] = {} + for index, field in enumerate(fields): + at: Loc = ("fields", index) + if field.name == "": + found.extend(problem((*at, "name"), "expected a non-empty field name", "invalid_value")) + first = seen.setdefault(field.name, index) + if first != index: + found.extend( + problem( + (*at, "name"), + f"duplicate field name {field.name!r}, already used by field {first}", + "invalid_value", + ) + ) + if ( + isinstance(field.data_type, DataTypeEntity) + and field.data_type.storage_class() == "variable_length" + ): + found.extend( + problem( + (*at, "data_type"), + "struct fields must use fixed-size data types", + "invalid_value", + ) + ) + return tuple(found) + + @dataclass(frozen=True) class StructDataType(DataTypeEntity): """The `struct` data type, coerced from its metadata. @@ -160,14 +211,16 @@ class StructDataType(DataTypeEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"fields": (True, _is_fields)} + value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @classmethod - def coerce(cls, value: object, context: "Context") -> Coerced[Self]: - struct, problems = super().coerce(value, context) - if struct is None: - return None, problems + def prepare( + cls, members: dict[str, object], context: "Context" + ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """Each field's data type, read in this scope.""" fields: list[StructFieldComponent] = [] found: list[ValidationProblem] = [] - for index, entry in enumerate(cast("tuple[object, ...]", struct.fields)): + for index, entry in enumerate(cast("tuple[object, ...]", members["fields"])): field = cast("Mapping[str, object]", entry) data_type, from_field = context.coerce( DATA_TYPE, field["data_type"], ("configuration", "fields", index, "data_type") @@ -176,7 +229,7 @@ def coerce(cls, value: object, context: "Context") -> Coerced[Self]: fields.append( StructFieldComponent(name=cast("str", field["name"]), data_type=data_type) ) - return replace(struct, fields=tuple(fields)), (*problems, *found) + return {**members, "fields": tuple(fields)}, tuple(found) def storage_class(self) -> StorageClass | None: """The widest class among the fields. @@ -198,47 +251,6 @@ def storage_class(self) -> StorageClass | None: widest = "multi_byte" return widest - def problems(self) -> tuple[ValidationProblem, ...]: - """What a struct can judge about its own fields. - - Names have to exist, be non-empty and be distinct, because a fill - value addresses fields by name. Field types have to be fixed-size, - because a record's layout is otherwise not determined. - """ - found: list[ValidationProblem] = [] - if len(self.fields) == 0: - found.extend( - problem(("fields",), "expected at least one struct field", "invalid_value") - ) - seen: dict[str, int] = {} - for index, field in enumerate(self.fields): - at: Loc = ("fields", index) - if field.name == "": - found.extend( - problem((*at, "name"), "expected a non-empty field name", "invalid_value") - ) - first = seen.setdefault(field.name, index) - if first != index: - found.extend( - problem( - (*at, "name"), - f"duplicate field name {field.name!r}, already used by field {first}", - "invalid_value", - ) - ) - if not isinstance(field.data_type, DataTypeEntity): - continue - if field.data_type.storage_class() == "variable_length": - found.extend( - problem( - (*at, "data_type"), - "struct fields must use fixed-size data types", - "invalid_value", - ) - ) - found.extend(within((*at, "data_type"), field.data_type.problems())) - return tuple(found) - def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """A fill value per field, addressed by name. diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index ab68f9ad23..3d4dbe05f6 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -781,11 +781,19 @@ def test_metadata_field_rejects_unknown_envelope_member() -> None: @pytest.mark.parametrize("field", ["codecs", "storage_transformers"]) -def test_optional_extension_points_allow_must_understand_false(field: str) -> None: - """Codecs and storage transformers may be explicitly ignorable.""" +def test_every_extension_point_rejects_must_understand_false(field: str) -> None: + """No extension point may be declared ignorable. + + Ignoring a codec gives wrong bytes as surely as ignoring a data type + gives wrong values, so `must_understand` is a property of the kind of + metadata rather than a per-occurrence choice. The spec names only the + three required points; this package reads that as an oversight. + """ doc: dict[str, object] = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc[field] = ({"name": "optional", "must_understand": False},) - assert validate_array_metadata_v3(doc) == () + assert [problem.loc for problem in validate_array_metadata_v3(doc)] == [ + (field, 0, "must_understand") + ] @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) diff --git a/packages/zarr-metadata/tests/rules/test_canonical.py b/packages/zarr-metadata/tests/rules/test_canonical.py index 16547752fe..ef7ed4dd61 100644 --- a/packages/zarr-metadata/tests/rules/test_canonical.py +++ b/packages/zarr-metadata/tests/rules/test_canonical.py @@ -81,10 +81,15 @@ def test_simplifies(overrides: dict[str, object], field: str, expected: object) assert _canonical(**overrides)[field] == expected -def test_an_explicit_must_understand_false_is_kept() -> None: - # `true` is the default and says nothing; `false` says something. - codec = {"name": "bytes", "must_understand": False} - assert _canonical(codecs=(codec,))["codecs"] == (codec,) +def test_error_an_extension_point_may_not_be_declared_ignorable() -> None: + # `must_understand` belongs to the kind of metadata, not to a use of + # it, and no extension point is skippable -- so there is nothing for + # canonicalization to keep. + result = canonicalize_array_metadata_v3( + {**BASE, "codecs": ({"name": "bytes", "must_understand": False},)} # type: ignore[arg-type] + ) + assert isinstance(result, Invalid) + assert [problem.loc for problem in result.problems] == [("codecs", 0, "must_understand")] def test_blosc_drops_a_typesize_that_shuffle_renders_ignored() -> None: diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index 2aa5bdda5a..2b2f19daa7 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -52,11 +52,14 @@ def _u(*lengths: int) -> tuple[frozenset[int], ...]: # that does not even pin the rank. GRIDS: dict[str, tuple[object, object, object, object]] = { "regular": (REGULAR, (64, 64), 2, _u(32, 32)), - "regular-zero-length-keeps-rank": ( + # A zero extent is not a grid, so the entity refuses to exist and + # only the rank the array shape pins survives -- the same answer as + # for a grid this package cannot read at all. + "regular-zero-length-is-unreadable": ( {"name": "regular", "configuration": {"chunk_shape": (0, 32)}}, (64, 64), 2, - (None, frozenset({32})), + (None, None), ), "rectilinear-uniform": (_rectilinear(((32, 32), (32, 32))), (64, 64), 2, _u(32, 32)), "rectilinear-uniform-rle": (_rectilinear((((32, 2),), ((32, 2),))), (64, 64), 2, _u(32, 32)), @@ -201,23 +204,22 @@ def test_error_index_codecs_are_judged_against_the_index_rank() -> None: ] -def test_error_a_bad_inner_extent_costs_that_axis_and_nothing_else() -> None: - # The zero is reported, and the inner pipeline is still judged against - # the rank the inner chunk shape declares. +def test_error_a_bad_inner_extent_costs_the_shard() -> None: + # A shard with a zero inner extent is not a shard, so it does not + # exist and nothing inside it is interpreted. The JSON is still there + # on the `Opaque` that replaces it; what is gone is the reading, and + # the one report that matters is the one you must fix first. inner = _shard((0, 2)) inner["configuration"] = { # type: ignore[index] **inner["configuration"], # type: ignore[dict-item] "codecs": ({"name": "transpose", "configuration": {"order": (0, 1, 2)}}, "bytes"), } - messages = [ - problem.message - for problem in validate_array_metadata_v3( - {**BASE, "data_type": "uint16", "chunk_grid": REGULAR, "codecs": (inner,)} - ) + problems = validate_array_metadata_v3( + {**BASE, "data_type": "uint16", "chunk_grid": REGULAR, "codecs": (inner,)} + ) + assert [problem.loc for problem in problems] == [ + ("codecs", 0, "configuration", "chunk_shape", 0) ] - assert any("positive chunk extent" in message for message in messages) - assert any("order has 3 entries" in message for message in messages) - assert any("endian is required" in message for message in messages) # An unmodelled codec is the ordinary case, not an exotic one: every diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 94c2fe43ea..3c53dca54f 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -599,9 +599,12 @@ def test_error_rank_is_judged_under_a_chunk_grid_with_unknown_extents() -> None: assert "2 dimensions" in message -def test_error_an_unusable_member_does_not_mask_the_rest_of_the_entity() -> None: - # A bad index_location says nothing about whether the inner pipelines - # are readable, so the pipeline problem must still be reported. +def test_error_an_unusable_member_costs_the_entity() -> None: + # An entity exists only if its members are readable and its values + # allowed, so a bad `index_location` means there is no shard to ask + # about its pipelines. The JSON survives on the `Opaque` that stands + # in for it; what is gone is the interpretation, and the report that + # remains is the one that has to be fixed first. problems = validate_array_metadata_v3( { **BASE, @@ -620,7 +623,6 @@ def test_error_an_unusable_member_does_not_mask_the_rest_of_the_entity() -> None ) assert {problem.loc for problem in problems} == { ("codecs", 0, "configuration", "index_location"), - ("codecs", 0, "configuration", "codecs", 1), } diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 62ac9a5dc1..0ba800e1ae 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -284,7 +284,7 @@ def test_every_problem_location_indexes_into_the_document() -> None: ), } problems = validate_array_metadata_v3(document) # type: ignore[arg-type] - assert len(problems) == 2 + assert len(problems) != 0 for problem in problems: node: object = document for step in problem.loc: @@ -297,10 +297,11 @@ def test_every_problem_location_indexes_into_the_document() -> None: node = node[step] # type: ignore[index] -def test_one_unreadable_member_does_not_hide_the_values_of_the_others() -> None: - # `clevel` is the wrong type, so this blosc cannot be built -- but - # `blocksize` was read, and what is wrong with it is still worth - # saying. Losing it would make fixing the document a two-pass job. +def test_an_unreadable_member_costs_the_entity() -> None: + # `clevel` is the wrong type, so this blosc cannot be built, and an + # entity that does not exist has no values to judge. The type problem + # is what you have to fix first, and the raw JSON is still on the + # `Opaque` standing in for the codec. document = { "zarr_format": 3, "node_type": "array", @@ -325,7 +326,6 @@ def test_one_unreadable_member_does_not_hide_the_values_of_the_others() -> None: problems = validate_array_metadata_v3(document) # type: ignore[arg-type] assert {problem.loc for problem in problems} == { ("codecs", 1, "configuration", "clevel"), - ("codecs", 1, "configuration", "blocksize"), } @@ -388,11 +388,10 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: }, ), "raw-bytes-padded": ("data_type", "r008"), - "scale-offset-explicit-null": ( + "scale-offset-scalar": ( "codecs", - {"name": "scale_offset", "configuration": {"offset": None}}, + {"name": "scale_offset", "configuration": {"offset": 2, "scale": 0.5}}, ), - "must-understand-false": ("codecs", {"name": "crc32c", "must_understand": False}), "struct-nested": ( "data_type", { @@ -473,3 +472,14 @@ def test_canonical_reaches_a_contained_entity() -> None: assert isinstance(shard, MetadataEntity) inner = shard.canonical().to_json()["configuration"]["codecs"][1] # type: ignore[index] assert "typesize" not in inner["configuration"] # type: ignore[index] + + +def test_error_an_explicit_null_scalar_is_refused() -> None: + # `null` is a value the document wrote, distinct from absence -- and + # no data type admits it as a scalar, so the codec cannot be built. + codec, problems = CORE_AND_EXTENSIONS.coerce( + "codecs", {"name": "scale_offset", "configuration": {"offset": None}} + ) + assert codec is not None + assert not isinstance(codec, MetadataEntity) + assert [problem.loc for problem in problems] == [("configuration", "offset")] diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index b145a0098d..ac0e6a0cc3 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -47,13 +47,15 @@ class AcmeLz4Codec(CodecEntity): variable_size: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"acceleration": (False, is_int)} - def problems(self) -> tuple[ValidationProblem, ...]: - if self.acceleration is UNSET: + @staticmethod + def value_problems(**members: object) -> tuple[ValidationProblem, ...]: + acceleration = members.get("acceleration", UNSET) + if acceleration is UNSET or not isinstance(acceleration, int): return () - if not 1 <= self.acceleration <= ACME_MAX_ACCELERATION: + if not 1 <= acceleration <= ACME_MAX_ACCELERATION: return problem( ("acceleration",), - f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {self.acceleration}", + f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {acceleration}", "invalid_value", ) return () @@ -255,3 +257,17 @@ def test_a_reader_can_choose_its_own_scope() -> None: in_scope = ArrayDocumentV3.from_json(document, context=SCOPE).codecs[1] assert isinstance(in_scope, AcmeLz4Codec) assert in_scope.acceleration == 4 + + +def test_error_value_rules_must_be_value_problems() -> None: + # `problems` was the old name and takes an entity; an override using + # it would never run, and nothing else would notice. + with pytest.raises(TypeError, match="value rules belong in `value_problems`"): + + @dataclass(frozen=True) + class Stale(CodecEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.stale" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def problems(self) -> tuple[ValidationProblem, ...]: + return () diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py index 5eb45b5fa2..37a18d1575 100644 --- a/packages/zarr-metadata/tests/v3/test_fill_values.py +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -10,6 +10,7 @@ import pytest from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS +from zarr_metadata.v3.entity import DataTypeEntity # (data type metadata, a fill value it accepts) ACCEPTED: dict[str, tuple[object, object]] = { @@ -25,7 +26,6 @@ "bytes-base64": ("bytes", "aGk="), "bytes-array": ("bytes", (1, 2, 3)), "raw-exact-width": ("r16", (0, 255)), - "raw-malformed-unjudged": ("r12", "anything"), "time-integer": ( {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, -1, @@ -92,6 +92,16 @@ def test_error_rejects(metadata: object, fill: object, reason: str) -> None: assert any(reason in problem.message for problem in problems), problems +def test_error_a_malformed_raw_name_has_no_entity_to_ask() -> None: + # `r12` is not a width, so the data type does not exist and there is + # nothing to put a fill value to. + entity, problems = CORE_AND_EXTENSIONS.coerce("data_type", "r12") + assert not isinstance(entity, DataTypeEntity) + assert [problem.message for problem in problems] == [ + "Expected 'r' where N is a positive multiple of 8, got 'r12'" + ] + + def test_an_unmodelled_data_type_judges_nothing() -> None: # Extension openness: a fill value we cannot interpret is not wrong. assert CORE_AND_EXTENSIONS.resolve("data_type", "mycorp.decimal") is None From 4e51814630096e03b4444bc41f2aa43de526ad9f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:03:32 +0200 Subject: [PATCH 046/107] fix(zarr-metadata): a required member has no default Defaults on required members were written for convenience while building and encode nothing -- `()` for a chunk shape, `Opaque(None, "invalid")` for a cast target, `r8` for a raw width. None is a spec default. Their cost was the construction invariant: `CastValueCodec()` built happily and serialized `{"data_type": null}` from a public constructor, and `TransposeCodec()` an empty permutation, neither through `unchecked`. Eighteen of them, gone. A conventional starting point is what `create_default` is for, as the model layer already spells it -- asking for one should be deliberate. `__init_subclass__` now enforces both halves: a required member has no default, an optional one defaults to UNSET. Fixing it also fixed a false positive it had -- it read `getattr`, so a member declared with `field(default=UNSET, kw_only=True)` was still a `Field` at hook time and was refused with a message accusing the author of the opposite of what they wrote. `@dataclass` runs after this hook, so the `Field` is unwrapped in place. `unchecked` gains the two checks it was missing. It dropped names it did not declare -- `BloscCodec.unchecked(clevle=9)` silently built with `clevel` at its default -- and, now that required members have no default, would otherwise leave an attribute unset, giving an entity whose `repr`, `==` and `hash` all raise. And the extension example in `v3.entity`'s docstring did not import: the guard that refuses it and the example were added in the same commit, and the example is the docs page every author copies first. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 66 +++++++++++++++++-- .../v3/chunk_grid/rectilinear.py | 4 +- .../zarr_metadata/v3/chunk_grid/regular.py | 2 +- .../src/zarr_metadata/v3/codec/blosc.py | 8 +-- .../src/zarr_metadata/v3/codec/cast_value.py | 6 +- .../src/zarr_metadata/v3/codec/gzip.py | 2 +- .../v3/codec/sharding_indexed.py | 6 +- .../src/zarr_metadata/v3/codec/transpose.py | 2 +- .../src/zarr_metadata/v3/codec/zstd.py | 2 +- .../v3/data_type/numpy_datetime64.py | 4 +- .../v3/data_type/numpy_timedelta64.py | 4 +- .../src/zarr_metadata/v3/data_type/raw.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 2 +- .../src/zarr_metadata/v3/entity.py | 4 +- 14 files changed, 82 insertions(+), 32 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 8178342f48..3fd94c9550 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -39,7 +39,7 @@ from __future__ import annotations from collections.abc import Mapping as _Mapping -from dataclasses import MISSING, dataclass, fields +from dataclasses import MISSING, Field, dataclass, fields from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast @@ -262,6 +262,10 @@ def coerce_members( """An entity's value-space judgment, over the members it was given.""" +_MISSING_DEFAULT: Final = object() +"""Distinguishes "declared no default" from a default that is None or UNSET.""" + + def _no_value_problems(**members: object) -> tuple[ValidationProblem, ...]: """An entity whose types admit only valid values has nothing to add.""" return () @@ -384,13 +388,31 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: if len(missing) != 0: msg = f"{cls.__name__} does not declare {', '.join(missing)}" raise TypeError(msg) + # A member's default decides whether the entity can exist without + # it, so the two kinds have opposite rules. `@dataclass` has not + # run yet, so a member declared with `field(...)` is still a + # `Field` here and its default has to be unwrapped. + defaulted: dict[str, object] = {} + for key in cls.member_types: + declared: object = getattr(cls, key, _MISSING_DEFAULT) + if type(declared) is Field: + # `field(...)`, so the default is inside it rather than + # being the attribute. `@dataclass` has not unwrapped it + # yet -- this hook runs first. + spec = cast("Field[object]", declared) + declared = ( + _MISSING_DEFAULT + if spec.default is MISSING and spec.default_factory is MISSING + else spec.default + ) + defaulted[key] = declared # An optional member defaults to UNSET or `configuration` emits it # for every instance, so the bare-name spelling becomes # unreachable and a document gains a member it never wrote. invented = [ key for key, (required, _) in cls.member_types.items() - if not required and getattr(cls, key, UNSET) is not UNSET + if not required and defaulted[key] is not UNSET ] if len(invented) != 0: msg = ( @@ -398,6 +420,21 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: f"{', '.join(invented)} a default other than UNSET" ) raise TypeError(msg) + # A required member with a default is an entity that can be built + # without it -- and then serializes a document nobody wrote. A + # conventional starting point is a `create_default` classmethod, + # named so that asking for one is deliberate. + presumed = [ + key + for key, (required, _) in cls.member_types.items() + if required and defaulted[key] is not _MISSING_DEFAULT + ] + if len(presumed) != 0: + msg = ( + f"{cls.__name__} gives the required member(s) " + f"{', '.join(presumed)} a default; required members have none" + ) + raise TypeError(msg) required_class_vars: ClassVar[tuple[str, ...]] = ("identifier",) """Every class variable a concrete entity of this kind must declare.""" @@ -542,15 +579,30 @@ def unchecked(cls, **members: object) -> Self: For a caller that has already asked -- `coerce` does, so that it can report the answer instead of raising it. Named so that choosing it is deliberate. + + Unchecked means *value*-unchecked. A member this entity does not + declare, or one with neither a value nor a default, is still a + `TypeError`: those produce an entity that cannot be repred, + compared or hashed, which no caller is asking for. """ + declared = {field_.name: field_ for field_ in fields(cls)} + unknown = sorted(members.keys() - declared.keys()) + if len(unknown) != 0: + msg = f"{cls.__name__} has no member(s) {', '.join(unknown)}" + raise TypeError(msg) entity = object.__new__(cls) - for field_ in fields(cls): - if field_.name in members: - object.__setattr__(entity, field_.name, members[field_.name]) + for name, field_ in declared.items(): + if name in members: + object.__setattr__(entity, name, members[name]) elif field_.default is not MISSING: - object.__setattr__(entity, field_.name, field_.default) + object.__setattr__(entity, name, field_.default) elif field_.default_factory is not MISSING: # pragma: no cover - none today - object.__setattr__(entity, field_.name, field_.default_factory()) + object.__setattr__(entity, name, field_.default_factory()) + else: + # Leaving it unset would give an entity whose `repr`, + # `==` and `hash` raise `AttributeError` on access. + msg = f"{cls.__name__} is missing a value for {name!r}" + raise TypeError(msg) return entity def to_json(self) -> ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index d2551b2aed..6447b88a9b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -244,8 +244,8 @@ def _value_problems( class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" - kind: Literal["inline"] = "inline" - chunk_shapes: tuple[RectilinearDimSpec, ...] = () + kind: Literal["inline"] + chunk_shapes: tuple[RectilinearDimSpec, ...] identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 8a9e9571bc..45522fef1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -89,7 +89,7 @@ def _value_problems( class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" - chunk_shape: tuple[int, ...] = () + chunk_shape: tuple[int, ...] identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index e58a16dc9b..396f6e9514 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -156,10 +156,10 @@ class BloscCodec(CodecEntity): equivalent document. """ - cname: BloscCName = "zstd" - clevel: int = 5 - shuffle: BloscShuffle = "noshuffle" - blocksize: int = 0 + cname: BloscCName + clevel: int + shuffle: BloscShuffle + blocksize: int typesize: int | UNSET = UNSET identifier: ClassVar[str] = BLOSC_CODEC_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index d42c90a6a4..341c638c5b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -174,10 +174,6 @@ def _is_data_type_field(value: object, loc: Loc) -> tuple[ValidationProblem, ... return () -_UNREAD: Final = Opaque(None, "invalid") -"""Placeholder for `data_type`, which is required and so never defaulted.""" - - @dataclass(frozen=True) class CastValueCodec(CodecEntity): """The `cast_value` codec, coerced from its metadata. @@ -186,7 +182,7 @@ class CastValueCodec(CodecEntity): read in a scope rather than on its own. """ - data_type: DataTypeEntity | Opaque = _UNREAD + data_type: DataTypeEntity | Opaque rounding: CastRoundingMode | UNSET = UNSET out_of_range: CastOutOfRangeMode | UNSET = UNSET scalar_map: ScalarMap | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 61404f3f35..1e55acb258 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -83,7 +83,7 @@ def _value_problems( class GzipCodec(CodecEntity): """The `gzip` codec, coerced from its metadata.""" - level: int = 5 + level: int identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index cdd3ab034a..89ff8dff02 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -181,9 +181,9 @@ class ShardingIndexedCodec(CodecEntity): itself an entity, read the same way this one was. """ - chunk_shape: tuple[int, ...] = () - codecs: tuple[CodecEntity | Opaque, ...] = () - index_codecs: tuple[CodecEntity | Opaque, ...] = () + chunk_shape: tuple[int, ...] + codecs: tuple[CodecEntity | Opaque, ...] + index_codecs: tuple[CodecEntity | Opaque, ...] index_location: ShardingIndexLocation | UNSET = UNSET identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 676a106f6b..5c178d31ac 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -88,7 +88,7 @@ def _value_problems( class TransposeCodec(CodecEntity): """The `transpose` codec, coerced from its metadata.""" - order: tuple[int, ...] = () + order: tuple[int, ...] identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME kind: ClassVar[CodecKind] = "array_array" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index b163f2abfb..b23487385a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -96,7 +96,7 @@ def _value_problems( class ZstdCodec(CodecEntity): """The `zstd` codec, coerced from its metadata.""" - level: int = 0 + level: int checksum: bool | UNSET = UNSET identifier: ClassVar[str] = ZSTD_CODEC_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 10bed5b53d..8d74618302 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -99,8 +99,8 @@ def _value_problems( class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" - unit: NumpyTimeUnit = "generic" - scale_factor: int = 1 + unit: NumpyTimeUnit + scale_factor: int scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index b6a31f2f3a..d70969a5af 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -119,8 +119,8 @@ def _value_problems( class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" - unit: NumpyTimeUnit = "generic" - scale_factor: int = 1 + unit: NumpyTimeUnit + scale_factor: int scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index fe583da854..d9a718ccfd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -111,7 +111,7 @@ class RawBytesDataType(DataTypeEntity): `r8`, and canonicalizing it away is not this package's call. """ - data_type_name: str = "r8" + data_type_name: str scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = RAW_BYTES_FAMILY diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 65db5f252c..f1201f59ed 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -203,7 +203,7 @@ class StructDataType(DataTypeEntity): read in to make sense of them. """ - fields: tuple[StructFieldComponent, ...] = () + fields: tuple[StructFieldComponent, ...] identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 62f6f5ea84..6318b54ee5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -30,7 +30,9 @@ @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): - acceleration: int = 1 + # A required member has no default; an optional one defaults to + # UNSET, so absence stays distinct from a JSON null. + acceleration: int | UNSET = UNSET identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" From 0e43ef65ea416b462219ca0e7632bb1da4a4e22f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:19:43 +0200 Subject: [PATCH 047/107] fix(zarr-metadata): must_understand: false is refused at every depth An extension point is something a reader must understand, and where the document wrote it does not change that. The envelope check refused the flag only at depth 0, so a `bytes` codec inside a shard's pipeline, or a `uint8` under a struct field, could declare itself ignorable and be read as if it had not -- the one case where the format's openness rule and the spec's requiredness rule disagree, decided silently in favour of the document. `Context.coerce` now passes `allow_must_understand_false=False`, which is the same judgment the model layer already gives a top-level field. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 2 +- .../src/zarr_metadata/v3/_registry.py | 7 ++- .../tests/rules/test_v3_array_rules.py | 50 +++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 3fd94c9550..d169222d16 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -491,7 +491,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # `typesize` requirement reads `shuffle`. Judging around the # hole would be guessing, so the type problems stand alone. return None, found - found = (*found, *within((), cls.value_problems(**members))) # type: ignore[arg-type] + found = (*found, *within((), cls.value_problems(**members))) if any(entry.kind != "unknown_key" for entry in found): return None, found # Already asked, so do not ask again on the way in. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index cb423975e4..bdcb6716aa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -203,7 +203,10 @@ def coerce( A metadata field is a metadata field wherever it appears, so the envelope gets the same structural judgment here that the model layer gives a top-level one -- an extra member, a `configuration` - that is not an object, a `must_understand` that is not a boolean. + that is not an object, a `must_understand` that is not a boolean + or is `false`. That last one is why the flag is passed: an + extension point is something a reader must understand at every + depth, not only at the document's top level. `envelope_judged` says that judgment has already happened, which it has for the fields of a document the model layer accepted. """ @@ -211,7 +214,7 @@ def coerce( if not envelope_judged: problems.extend( ValidationProblem((*loc, *found.loc), found.message, found.kind) - for found in validate_metadata_field_v3(value) + for found in validate_metadata_field_v3(value, allow_must_understand_false=False) ) name, _, _ = named_configuration(value) if name is None: diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 3c53dca54f..fcb7338225 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -744,3 +744,53 @@ def test_zstd_level_range(level: int, valid: bool) -> None: assert problems == () else: assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "level")] + + +# -- must_understand: false, wherever a document writes it -------------------- + +# (a document nesting an ignorable extension point, and where it sits) +IGNORABLE_NESTED: dict[str, tuple[Mapping[str, object], tuple[object, ...]]] = { + "inner-codec-of-a-shard": ( + { + **BASE, + "codecs": (_shard(codecs=({"name": "bytes", "must_understand": False},)),), + }, + ("codecs", 0, "configuration", "codecs", 0, "must_understand"), + ), + "index-codec-of-a-shard": ( + { + **BASE, + "codecs": (_shard(index_codecs=({**_INDEX_BYTES, "must_understand": False},)),), + }, + ("codecs", 0, "configuration", "index_codecs", 0, "must_understand"), + ), + "data-type-of-a-struct-field": ( + { + **BASE, + "data_type": { + "name": "struct", + "configuration": { + "fields": ( + { + "name": "a", + "data_type": {"name": "uint8", "must_understand": False}, + }, + ) + }, + }, + "fill_value": {"a": 0}, + }, + ("data_type", "configuration", "fields", 0, "data_type", "must_understand"), + ), +} + + +@pytest.mark.parametrize("case", IGNORABLE_NESTED) +def test_error_a_nested_extension_point_may_not_be_declared_ignorable(case: str) -> None: + # An extension point is something a reader must understand, and depth + # does not change that: a codec inside a shard is still a codec. The + # top-level check would otherwise be a check on where the flag was + # written rather than on what it says. + document, loc = IGNORABLE_NESTED[case] + problems = validate_array_metadata_v3(cast("Any", document)) + assert loc in {problem.loc for problem in problems} From 24fe9e46daa73ae6147f8d9f3d0229a86e25afbc Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:21:21 +0200 Subject: [PATCH 048/107] fix(zarr-metadata): to_json hands back nothing the entity still holds `configuration()` returned the entity's own member dict, so the document `to_json` built shared structure with the entity that built it. A member can be an arbitrary JSON value -- a `scale_offset` offset may be an object, a `cast_value` scalar map is one, a `struct`'s fields are a tuple of them -- and mutating what the caller was handed reached back into a frozen entity. Frozen is a claim about the whole value, not about the outermost binding. Deep-copied on the way out, which is the guarantee the model layer already makes for its own round-trips. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_entity.py | 8 +++- .../zarr-metadata/tests/v3/test_entities.py | 42 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index d169222d16..1d8b01124f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -39,6 +39,7 @@ from __future__ import annotations from collections.abc import Mapping as _Mapping +from copy import deepcopy from dataclasses import MISSING, Field, dataclass, fields from types import MappingProxyType from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast @@ -526,8 +527,13 @@ def configuration(self) -> dict[str, object]: this package holds `None` to mean a JSON `null` the document actually wrote, and `scale_offset` is a real case where `null` and absent are different documents. + + Deep-copied, because a member can be an arbitrary JSON value: a + `scale_offset` offset may be an object, and handing the caller + the entity's own dict would let them mutate a frozen entity + through the document it returned. """ - return self._members() + return deepcopy(self._members()) value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) """Every value among the members the spec disallows. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 0ba800e1ae..9462c766c8 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -9,6 +9,7 @@ from __future__ import annotations +import copy import dataclasses from typing import get_type_hints @@ -483,3 +484,44 @@ def test_error_an_explicit_null_scalar_is_refused() -> None: assert codec is not None assert not isinstance(codec, MetadataEntity) assert [problem.loc for problem in problems] == [("configuration", "offset")] + + +# (an entity whose configuration holds a mutable JSON value) +MUTABLE_MEMBERS: dict[str, tuple[str, object]] = { + "scale-offset-object": ( + "codecs", + {"name": "scale_offset", "configuration": {"offset": {"a": 1}}}, + ), + "cast-value-scalar-map": ( + "codecs", + { + "name": "cast_value", + "configuration": {"data_type": "int8", "scalar_map": {"encode": (("NaN", 0),)}}, + }, + ), + "struct-fields": ( + "data_type", + {"name": "struct", "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}}, + ), +} + + +@pytest.mark.parametrize(("field", "written"), MUTABLE_MEMBERS.values(), ids=list(MUTABLE_MEMBERS)) +def test_to_json_shares_no_mutable_state_with_the_entity(field: str, written: object) -> None: + # The model layer has this test; the entity layer did not, and handed + # out its own dict -- so a caller mutating the document it was given + # mutated a frozen entity. + entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) # type: ignore[arg-type] + assert problems == () + assert isinstance(entity, MetadataEntity) + baseline = copy.deepcopy(entity.to_json()) + handed_out = entity.to_json() + configuration = handed_out["configuration"] # type: ignore[index] + assert isinstance(configuration, dict) + for key in list(configuration): + value = configuration[key] + if isinstance(value, dict): + value["INJECTED"] = "boom" + else: + configuration[key] = "clobbered" + assert entity.to_json() == baseline From a6a62de1e658b22b8f54001f243ee3ece0ac1f2c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:24:35 +0200 Subject: [PATCH 049/107] refactor(zarr-metadata): a table of entities knows which point it is `Context.entities` was `Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]]`, which is true of a scope that files a codec under `data_type`. Nothing said otherwise until something asked that codec for a storage class -- registration succeeded, resolution succeeded, and the failure surfaced somewhere else entirely. `EntityTables` gives each point its own entity kind, so a misfiling is an error where the table is written; `_ENTITY_KINDS` says the same at run time, for a scope assembled from an entry point or from configuration, where there was no type to check. `resolve` gains the per-point overloads `coerce` already had, which is what the kinds were for. `tables()` widens the five to the kind they share, once and by hand, so nothing needs a `cast` to iterate them. `extended_with` is how a reader registers entities of its own: a per- point merge, so naming one point leaves the rest of the scope alone. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../src/zarr_metadata/v3/_registry.py | 146 ++++++++++++++++-- .../src/zarr_metadata/v3/entity.py | 12 +- .../zarr-metadata/tests/test_public_api.py | 3 + .../zarr-metadata/tests/v3/test_entities.py | 7 +- .../tests/v3/test_extension_api.py | 30 ++-- 5 files changed, 170 insertions(+), 28 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index bdcb6716aa..24e2d67f0f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -23,14 +23,18 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, overload +from typing_extensions import TypedDict, Unpack + from zarr_metadata.model._validation import ( ValidationProblem, validate_metadata_field_v3, ) from zarr_metadata.v3._entity import ( + STORAGE_TRANSFORMERS, ChunkGridEntity, CodecEntity, DataTypeEntity, + MetadataEntity, Opaque, named_configuration, ) @@ -78,10 +82,59 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.v3._entity import Loc, MetadataEntity + from zarr_metadata.v3._entity import Loc from zarr_metadata.v3._extension_points import ExtensionPointField +class EntityTables(TypedDict): + """Which entity is registered under which name, at each extension point. + + Typed per point rather than as one mapping, because an entity's kind + is a fact about where it may be registered: a codec at `data_type` + would satisfy a `Mapping[str, type[MetadataEntity]]` and then fail + the moment anything asked it for a storage class. Spelling the + correspondence here is what makes `resolve`'s per-point return type + true rather than asserted, and what turns a misfiling into an error + where the table is written. + """ + + data_type: Mapping[str, type[DataTypeEntity]] + codecs: Mapping[str, type[CodecEntity]] + chunk_grid: Mapping[str, type[ChunkGridEntity]] + chunk_key_encoding: Mapping[str, type[MetadataEntity]] + storage_transformers: Mapping[str, type[MetadataEntity]] + + +class PartialEntityTables(TypedDict, total=False): + """`EntityTables` with every point optional: what `extended_with` takes. + + A reader registering a codec of its own says so and nothing else; + the points it does not name keep whatever the scope it extended had. + """ + + data_type: Mapping[str, type[DataTypeEntity]] + codecs: Mapping[str, type[CodecEntity]] + chunk_grid: Mapping[str, type[ChunkGridEntity]] + chunk_key_encoding: Mapping[str, type[MetadataEntity]] + storage_transformers: Mapping[str, type[MetadataEntity]] + + +_ENTITY_KINDS: Final[Mapping[ExtensionPointField, type[MetadataEntity]]] = { + DATA_TYPE: DataTypeEntity, + CODECS: CodecEntity, + CHUNK_GRID: ChunkGridEntity, + CHUNK_KEY_ENCODING: MetadataEntity, + STORAGE_TRANSFORMERS: MetadataEntity, +} +"""The base every entity at a point must derive from. + +`EntityTables` says the same thing to the type checker, which is where a +table written out in source is caught. This is for the one built at run +time -- from a plugin entry point, from configuration -- where there was +no type to check. +""" + + @dataclass(frozen=True, slots=True) class Context: """The entities in scope while metadata is being read. @@ -98,10 +151,15 @@ class Context: specification plus what `zarr-extensions` registers. """ - entities: Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]] + entities: EntityTables def __post_init__(self) -> None: - """Refuse a table whose key an entity would not answer to. + """Refuse a table an entity does not belong in. + + Two ways it can fail to. The entity may be of the wrong kind for + the point -- a codec under `data_type` -- which `EntityTables` + catches in source and this catches in a scope assembled at run + time. Or its key may not be its identifier. `resolve` finds a candidate by key and then asks the entity whether the name is really one of its own, so a key that is not @@ -115,8 +173,14 @@ def __post_init__(self) -> None: raw-bytes family registers under an invented one that `accepts` deliberately refuses. """ - for field, table in self.entities.items(): + for field, table in self.tables().items(): for key, entity in table.items(): + if not issubclass(entity, _ENTITY_KINDS[field]): + msg = ( + f"{entity.__name__} is registered at {field!r}, which takes " + f"{_ENTITY_KINDS[field].__name__} entities" + ) + raise TypeError(msg) if key != entity.identifier: msg = ( f"{entity.__name__} is registered at {field!r} under {key!r} " @@ -124,6 +188,62 @@ def __post_init__(self) -> None: ) raise ValueError(msg) + def extended_with(self, **entities: Unpack[PartialEntityTables]) -> Context: + """This scope, plus entities of your own at the points named. + + The merge is per point, so naming `codecs` adds codecs rather + than replacing the ones already in scope. A name already + registered is taken over by what is passed here, which is how a + reader substitutes its own reading of a codec the package + already models. + """ + return Context( + { + DATA_TYPE: {**self.entities["data_type"], **entities.get("data_type", {})}, + CODECS: {**self.entities["codecs"], **entities.get("codecs", {})}, + CHUNK_GRID: {**self.entities["chunk_grid"], **entities.get("chunk_grid", {})}, + CHUNK_KEY_ENCODING: { + **self.entities["chunk_key_encoding"], + **entities.get("chunk_key_encoding", {}), + }, + STORAGE_TRANSFORMERS: { + **self.entities["storage_transformers"], + **entities.get("storage_transformers", {}), + }, + } + ) + + def tables(self) -> Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]]: + """Every point's table, under the one kind all entities share. + + `entities` gives each point its own entity kind, which is the + point of it, and a key that is not a literal loses that. So the + widening is written out here, once, by hand rather than asserted + with a `cast`: reading each member by its own key is what makes + the result checked rather than promised. Anything that asks the + scope what is in it, rather than asking it about one point, wants + this. + """ + return { + DATA_TYPE: self.entities["data_type"], + CODECS: self.entities["codecs"], + CHUNK_GRID: self.entities["chunk_grid"], + CHUNK_KEY_ENCODING: self.entities["chunk_key_encoding"], + STORAGE_TRANSFORMERS: self.entities["storage_transformers"], + } + + @overload + def resolve(self, field: Literal["data_type"], name: str) -> type[DataTypeEntity] | None: ... + + @overload + def resolve(self, field: Literal["codecs"], name: str) -> type[CodecEntity] | None: ... + + @overload + def resolve(self, field: Literal["chunk_grid"], name: str) -> type[ChunkGridEntity] | None: ... + + @overload + def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: ... + def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: """The entity `name` denotes at `field`, or None if out of scope. @@ -136,7 +256,7 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] really one of its own. Otherwise the identifier itself would be a name a document could write. """ - entity = self.entities.get(field, {}).get(canonical_name(field, name)) + entity = self.tables()[field].get(canonical_name(field, name)) if entity is None or not entity.accepts(name): return None return entity @@ -234,7 +354,7 @@ def coerce( return entity, tuple(problems) -_CORE_CODECS: Final[dict[str, type[MetadataEntity]]] = { +_CORE_CODECS: Final[dict[str, type[CodecEntity]]] = { BloscCodec.identifier: BloscCodec, BytesCodec.identifier: BytesCodec, Crc32cCodec.identifier: Crc32cCodec, @@ -242,13 +362,13 @@ def coerce( ShardingIndexedCodec.identifier: ShardingIndexedCodec, TransposeCodec.identifier: TransposeCodec, } -_EXTENSION_CODECS: Final[dict[str, type[MetadataEntity]]] = { +_EXTENSION_CODECS: Final[dict[str, type[CodecEntity]]] = { CastValueCodec.identifier: CastValueCodec, ScaleOffsetCodec.identifier: ScaleOffsetCodec, ZstdCodec.identifier: ZstdCodec, } -_CORE_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = { +_CORE_DATA_TYPES: Final[dict[str, type[DataTypeEntity]]] = { BoolDataType.identifier: BoolDataType, Int8DataType.identifier: Int8DataType, Int16DataType.identifier: Int16DataType, @@ -265,7 +385,7 @@ def coerce( Complex128DataType.identifier: Complex128DataType, RawBytesDataType.identifier: RawBytesDataType, } -_EXTENSION_DATA_TYPES: Final[dict[str, type[MetadataEntity]]] = { +_EXTENSION_DATA_TYPES: Final[dict[str, type[DataTypeEntity]]] = { BytesDataType.identifier: BytesDataType, StringDataType.identifier: StringDataType, NumpyDatetime64DataType.identifier: NumpyDatetime64DataType, @@ -273,10 +393,10 @@ def coerce( StructDataType.identifier: StructDataType, } -_CORE_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = { +_CORE_CHUNK_GRIDS: Final[dict[str, type[ChunkGridEntity]]] = { RegularChunkGrid.identifier: RegularChunkGrid, } -_EXTENSION_CHUNK_GRIDS: Final[dict[str, type[MetadataEntity]]] = { +_EXTENSION_CHUNK_GRIDS: Final[dict[str, type[ChunkGridEntity]]] = { RectilinearChunkGrid.identifier: RectilinearChunkGrid, } @@ -292,6 +412,7 @@ def coerce( DATA_TYPE: _CORE_DATA_TYPES, CHUNK_GRID: _CORE_CHUNK_GRIDS, CHUNK_KEY_ENCODING: _CORE_CHUNK_KEY_ENCODINGS, + STORAGE_TRANSFORMERS: {}, } ) """Only what the Zarr v3 specification defines.""" @@ -302,6 +423,7 @@ def coerce( DATA_TYPE: {**_CORE_DATA_TYPES, **_EXTENSION_DATA_TYPES}, CHUNK_GRID: {**_CORE_CHUNK_GRIDS, **_EXTENSION_CHUNK_GRIDS}, CHUNK_KEY_ENCODING: {**_CORE_CHUNK_KEY_ENCODINGS}, + STORAGE_TRANSFORMERS: {}, } ) """What the specification defines, plus what `zarr-extensions` registers.""" @@ -311,4 +433,6 @@ def coerce( "CORE", "CORE_AND_EXTENSIONS", "Context", + "EntityTables", + "PartialEntityTables", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 6318b54ee5..fc8b648ecf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -91,6 +91,7 @@ class AcmeLz4Codec(CodecEntity): Opaque, StorageClass, TypeCheck, + ValueRoutine, coerce_members, is_bool, is_int, @@ -104,7 +105,13 @@ class AcmeLz4Codec(CodecEntity): within, ) from zarr_metadata.v3._parts import UNKNOWN_GRID, ArrayParts, ChunkGrid, Extents, shard_index_grid -from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context +from zarr_metadata.v3._registry import ( + CORE, + CORE_AND_EXTENSIONS, + Context, + EntityTables, + PartialEntityTables, +) from zarr_metadata.v3.data_type._families import ( FLOAT_SPECIALS, ComplexDataType, @@ -135,6 +142,7 @@ class AcmeLz4Codec(CodecEntity): "ComplexDataType", "Context", "DataTypeEntity", + "EntityTables", "ExtensionPointField", "Extents", "FloatDataType", @@ -144,8 +152,10 @@ class AcmeLz4Codec(CodecEntity): "MetadataEntity", "NumpyTimeDataType", "Opaque", + "PartialEntityTables", "StorageClass", "TypeCheck", + "ValueRoutine", "array_problems_v3", "as_sequence", "byte_values", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 584ebce6bd..913e18c69d 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -303,6 +303,7 @@ def test_all_is_grouped_and_unique() -> None: "ArrayParts", "ArrayDocumentV3", "Endianness", + "EntityTables", "Invalid", "HexFloat16", "HexFloat32", @@ -310,6 +311,7 @@ def test_all_is_grouped_and_unique() -> None: "JSONValue", "MetadataValidationError", "Opaque", + "PartialEntityTables", "NumpyDatetime64", "NumpyTimeUnit", "NumpyTimedelta64", @@ -321,6 +323,7 @@ def test_all_is_grouped_and_unique() -> None: "Struct", "StructField", "ValidationProblem", + "ValueRoutine", } ) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 9462c766c8..17a7d1e4b3 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -165,15 +165,16 @@ def test_a_required_member_rules_out_the_bare_spelling( def test_every_registered_entity_is_checked_here() -> None: registered = { f"{field}:{identifier}" - for field, entities in CORE_AND_EXTENSIONS.entities.items() + for field, entities in CORE_AND_EXTENSIONS.tables().items() for identifier in entities } assert registered == set(CONFIGURATIONS) def test_core_is_a_subset_of_core_and_extensions() -> None: - for field, entities in CORE.entities.items(): - assert entities.items() <= CORE_AND_EXTENSIONS.entities[field].items() + both = CORE_AND_EXTENSIONS.tables() + for field, entities in CORE.tables().items(): + assert entities.items() <= both[field].items() def test_a_name_out_of_scope_resolves_to_nothing() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index ac0e6a0cc3..3a7db5e810 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -17,6 +17,7 @@ validate_array_metadata_v3, ) from zarr_metadata.v3.entity import ( + CORE, CORE_AND_EXTENSIONS, ArrayDocumentV3, ArrayParts, @@ -70,16 +71,9 @@ class AcmeFloat8DataType(DataTypeEntity): def _scope() -> Context: - entities = dict(CORE_AND_EXTENSIONS.entities) - return Context( - { - **entities, - "codecs": {**entities["codecs"], AcmeLz4Codec.identifier: AcmeLz4Codec}, - "data_type": { - **entities["data_type"], - AcmeFloat8DataType.identifier: AcmeFloat8DataType, - }, - } + return CORE_AND_EXTENSIONS.extended_with( + codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}, + data_type={AcmeFloat8DataType.identifier: AcmeFloat8DataType}, ) @@ -163,7 +157,17 @@ def test_error_a_registry_key_must_be_the_identifier() -> None: # Otherwise `resolve` never finds it and the document is silently # waved through, indistinguishable from openness. with pytest.raises(ValueError, match="registered at 'codecs' under 'acme.lz-4'"): - Context({"codecs": {"acme.lz-4": AcmeLz4Codec}}) + CORE.extended_with(codecs={"acme.lz-4": AcmeLz4Codec}) + + +def test_error_an_entity_cannot_be_registered_at_the_wrong_point() -> None: + # `EntityTables` says so to the type checker, which settles a scope + # written out in source. A scope assembled at run time -- from an + # entry point, from configuration -- had no type to check, and a + # codec under `data_type` would resolve and then be asked for a + # storage class it has no answer to. + with pytest.raises(TypeError, match="registered at 'data_type', which takes DataTypeEntity"): + CORE.extended_with(data_type={AcmeLz4Codec.identifier: AcmeLz4Codec}) # type: ignore[dict-item] def test_the_entity_layer_answers_what_a_reader_needs() -> None: @@ -177,8 +181,8 @@ def test_the_entity_layer_answers_what_a_reader_needs() -> None: "chunk_grid", {"name": "regular", "configuration": {"chunk_shape": (32, 32)}} ) assert problems == () - assert isinstance(grid, MetadataEntity) - parts = ArrayParts(grid.grid((64, 64)), data_type) # type: ignore[attr-defined] + assert isinstance(grid, ChunkGridEntity) + parts = ArrayParts(grid.grid((64, 64)), data_type) assert parts.grid.rank == 2 assert parts.grid.axis(0) == frozenset({32}) From a71de2ccdef0eee8dd199e790824b6793150242b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:25:38 +0200 Subject: [PATCH 050/107] refactor(zarr-metadata): a codec declares its kind, so no table does `codec.kind` sorted codec names into the spec's three pipeline kinds by listing them, which put the same fact in two places and made the module import every codec to build tuples the codecs themselves already knew. A name does not have a pipeline position; a codec does. Removed with it: `codec_kind_of_name`, the three name tuples, and `codec.blosc.canonical_configuration`, whose one job is now `BloscCodec.canonical()`. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/changes/4379.misc.1.md | 8 +++ .../src/zarr_metadata/v3/_entity.py | 5 +- .../src/zarr_metadata/v3/codec/__init__.py | 16 +---- .../src/zarr_metadata/v3/codec/blosc.py | 17 ----- .../src/zarr_metadata/v3/codec/kind.py | 64 ------------------- .../tests/rules/test_chain_properties.py | 7 +- .../zarr-metadata/tests/v3/codec/test_kind.py | 26 -------- 7 files changed, 18 insertions(+), 125 deletions(-) delete mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py delete mode 100644 packages/zarr-metadata/tests/v3/codec/test_kind.py diff --git a/packages/zarr-metadata/changes/4379.misc.1.md b/packages/zarr-metadata/changes/4379.misc.1.md index 3cdfba2401..61efd43903 100644 --- a/packages/zarr-metadata/changes/4379.misc.1.md +++ b/packages/zarr-metadata/changes/4379.misc.1.md @@ -15,3 +15,11 @@ What replaces the drift tests is a correspondence test: an entity's dataclass fields, its configuration TypedDict, and its member table are three spellings of one set, and whether the bare-name spelling is allowed follows from the TypedDict's required keys. + +One more table went the same way: `zarr_metadata.v3.codec.kind`, which +sorted codec *names* into the spec's three pipeline kinds. A name does +not have a pipeline position, a codec does, and each one now declares +its own `kind`. Removed with it: `codec_kind_of_name`, +`ARRAY_ARRAY_CODEC_NAMES`, `ARRAY_BYTES_CODEC_NAMES`, +`BYTES_BYTES_CODEC_NAMES`, and `codec.blosc.canonical_configuration`, +whose one job is `BloscCodec.canonical()`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 1d8b01124f..8523c76176 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -114,9 +114,8 @@ CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] """The three pipeline positions the v3 spec sorts codecs into. -Here rather than in `zarr_metadata.v3.codec.kind` because each codec -declares its own kind, and that module imports every codec to build the -tuples it will no longer need once they all do. +Declared by each codec, which is why there is no table of it: a name +does not have a pipeline position, a codec does. """ TypeCheck: TypeAlias = "Callable[[object, Loc], tuple[ValidationProblem, ...]]" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index f22a2280f9..05b484eca8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -14,33 +14,24 @@ `codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. -The `kind` submodule sorts the known codec names into the spec's three -pipeline kinds (`array -> array`, `array -> bytes`, `bytes -> bytes`). +Each codec declares its own pipeline kind (`array -> array`, +`array -> bytes`, `bytes -> bytes`) as a `kind` class variable. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ +from zarr_metadata.v3._entity import CodecKind from zarr_metadata.v3.codec.blosc import BloscCodecMetadata from zarr_metadata.v3.codec.bytes import BytesCodecMetadata from zarr_metadata.v3.codec.cast_value import CastValueCodecMetadata from zarr_metadata.v3.codec.crc32c import Crc32cCodecMetadata from zarr_metadata.v3.codec.gzip import GzipCodecMetadata -from zarr_metadata.v3.codec.kind import ( - ARRAY_ARRAY_CODEC_NAMES, - ARRAY_BYTES_CODEC_NAMES, - BYTES_BYTES_CODEC_NAMES, - CodecKind, - codec_kind_of_name, -) from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodecMetadata from zarr_metadata.v3.codec.sharding_indexed import ShardingIndexedCodecMetadata from zarr_metadata.v3.codec.transpose import TransposeCodecMetadata from zarr_metadata.v3.codec.zstd import ZstdCodecMetadata __all__ = [ - "ARRAY_ARRAY_CODEC_NAMES", - "ARRAY_BYTES_CODEC_NAMES", - "BYTES_BYTES_CODEC_NAMES", "BloscCodecMetadata", "BytesCodecMetadata", "CastValueCodecMetadata", @@ -51,5 +42,4 @@ "ShardingIndexedCodecMetadata", "TransposeCodecMetadata", "ZstdCodecMetadata", - "codec_kind_of_name", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 396f6e9514..6b3fd73f8d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -4,7 +4,6 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html """ -from collections.abc import Mapping from dataclasses import dataclass, replace from typing import ClassVar, Final, Literal, NotRequired, Self, cast @@ -78,21 +77,6 @@ class BloscCodecObject(TypedDict, closed=True): """ -def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, object]: - """A blosc configuration in its simplest equivalent form. - - Under `shuffle: "noshuffle"` the spec says of `typesize` that "the - value is ignored", so whatever it holds carries no meaning and two - documents differing only there describe the same codec. Dropping it - makes that equality visible. - - Assumes a configuration the shape validator has already accepted. - """ - if configuration.get("shuffle") != BLOSC_NO_SHUFFLE or "typesize" not in configuration: - return configuration - return {key: value for key, value in configuration.items() if key != "typesize"} - - __all__ = [ "BLOSC_CNAME", "BLOSC_CODEC_NAME", @@ -105,7 +89,6 @@ def canonical_configuration(configuration: Mapping[str, object]) -> Mapping[str, "BloscCodecName", "BloscCodecObject", "BloscShuffle", - "canonical_configuration", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py deleted file mode 100644 index 6578719cea..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Classify Zarr v3 codecs by pipeline kind. - -The v3 spec sorts codecs into three kinds — `array -> array`, -`array -> bytes`, `bytes -> bytes` — and a pipeline is -`array->array* array->bytes bytes->bytes*`. `codec_kind_of_name` -classifies a known name; unknown names have no kind. - -See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html -""" - -from typing import Final - -from zarr_metadata.v3._entity import CodecKind -from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME -from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME -from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME -from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME -from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME -from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME -from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC_NAME -from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME -from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME - -ARRAY_ARRAY_CODEC_NAMES: Final = ( - TRANSPOSE_CODEC_NAME, - CAST_VALUE_CODEC_NAME, - SCALE_OFFSET_CODEC_NAME, -) -"""Tuple of the `name` field values of the known `array -> array` codecs.""" - -ARRAY_BYTES_CODEC_NAMES: Final = (BYTES_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME) -"""Tuple of the `name` field values of the known `array -> bytes` codecs.""" - -BYTES_BYTES_CODEC_NAMES: Final = ( - BLOSC_CODEC_NAME, - CRC32C_CODEC_NAME, - GZIP_CODEC_NAME, - ZSTD_CODEC_NAME, -) -"""Tuple of the `name` field values of the known `bytes -> bytes` codecs.""" - - -def codec_kind_of_name(name: str) -> CodecKind | None: - """The pipeline kind of the codec named `name`, or None if unknown. - - Classifies by name alone, with no judgment of the entry's spelling or - configuration; the rules layer judges those separately. - """ - if name in ARRAY_ARRAY_CODEC_NAMES: - return "array_array" - if name in ARRAY_BYTES_CODEC_NAMES: - return "array_bytes" - if name in BYTES_BYTES_CODEC_NAMES: - return "bytes_bytes" - return None - - -__all__ = [ - "ARRAY_ARRAY_CODEC_NAMES", - "ARRAY_BYTES_CODEC_NAMES", - "BYTES_BYTES_CODEC_NAMES", - "CodecKind", - "codec_kind_of_name", -] diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py index d50dd6482c..7ddf72309e 100644 --- a/packages/zarr-metadata/tests/rules/test_chain_properties.py +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -27,7 +27,7 @@ valid_documents, ) from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.v3.codec.kind import codec_kind_of_name +from zarr_metadata.v3.entity import CODECS, CORE_AND_EXTENSIONS if TYPE_CHECKING: from collections.abc import Mapping @@ -60,7 +60,10 @@ def test_the_strategies_cover_every_codec_the_package_models() -> None: assert modelled == drawn for kinds, expected in ((ARRAY_ARRAY, "array_array"), (ARRAY_BYTES, "array_bytes")): for entry in kinds: - assert codec_kind_of_name(entry.__annotations__["name"].__args__[0]) == expected + name = entry.__annotations__["name"].__args__[0] + entity = CORE_AND_EXTENSIONS.resolve(CODECS, name) + assert entity is not None, name + assert entity.kind == expected @given(codec_chains()) diff --git a/packages/zarr-metadata/tests/v3/codec/test_kind.py b/packages/zarr-metadata/tests/v3/codec/test_kind.py deleted file mode 100644 index 996d469035..0000000000 --- a/packages/zarr-metadata/tests/v3/codec/test_kind.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Tests for codec kind classification.""" - -from __future__ import annotations - -import pytest - -from zarr_metadata.v3.codec.kind import codec_kind_of_name - -# (codec name, expected kind). Classification is by name alone. -CASES: dict[str, str | None] = { - "transpose": "array_array", - "cast_value": "array_array", - "scale_offset": "array_array", - "bytes": "array_bytes", - "sharding_indexed": "array_bytes", - "blosc": "bytes_bytes", - "crc32c": "bytes_bytes", - "gzip": "bytes_bytes", - "zstd": "bytes_bytes", - "lightspeed": None, -} - - -@pytest.mark.parametrize(("name", "kind"), CASES.items(), ids=list(CASES)) -def test_kind_of_name(name: str, kind: str | None) -> None: - assert codec_kind_of_name(name) == kind From a0e10381f639e84c3b751168477ee69ea16ae363 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:29:20 +0200 Subject: [PATCH 051/107] refactor(zarr-metadata): a value routine says which members it judges `value_problems` was a module-level function assigned to a `ClassVar[ValueRoutine]`, and the declared type is what a caller sees: `(...) -> tuple[ValidationProblem, ...]`, which accepts any arguments at all. `GzipCodec.value_problems(levle="three")` type-checked. Defined in the class as a `@staticmethod` instead, the entity's own declaration wins and the signature survives -- pyright reads `BloscCodec.value_problems` as `(**members: **BloscCodecConfiguration)` and refuses a member that is missing, misspelled or of the wrong type. The base keeps the open `ClassVar`, which is what makes overriding it legal. It also puts the routine next to the class it is about. `coerce` calls it as `value_problems(**members)` over a dict built at run time, which no signature can check, so the correspondence is a test: the routine's TypedDict names exactly the entity's own fields. A `struct` and a `sharding_indexed` annotate a TypedDict of their own, because `prepare` has replaced field objects with entities by then -- that changes the member types, never which members there are. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.10.md | 14 +-- .../v3/chunk_grid/rectilinear.py | 79 +++++++++-------- .../zarr_metadata/v3/chunk_grid/regular.py | 43 +++++---- .../src/zarr_metadata/v3/codec/blosc.py | 81 ++++++++--------- .../src/zarr_metadata/v3/codec/gzip.py | 23 +++-- .../zarr_metadata/v3/codec/scale_offset.py | 43 +++++---- .../v3/codec/sharding_indexed.py | 39 ++++----- .../src/zarr_metadata/v3/codec/transpose.py | 37 ++++---- .../src/zarr_metadata/v3/codec/zstd.py | 29 +++---- .../v3/data_type/numpy_datetime64.py | 35 ++++---- .../v3/data_type/numpy_timedelta64.py | 35 ++++---- .../src/zarr_metadata/v3/data_type/struct.py | 87 ++++++++++--------- .../zarr-metadata/tests/v3/test_entities.py | 23 ++++- 13 files changed, 286 insertions(+), 282 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index 66768fbedc..bc9ba73ab4 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -5,11 +5,15 @@ such thing as an invalid entity to hold. Code that wants unvalidated metadata already has somewhere to put it: `Opaque`, which carries the JSON verbatim. -Three pieces, as the structure requires. `value_problems` is a routine -over the members rather than a method on an entity, because judging -values does not need one -- and needing one would mean an invalid one had -been built. `unchecked` builds without asking, for a caller that has -already asked. The dataclass constructor asks, then builds. +Three pieces, as the structure requires. `value_problems` is a static +routine over the members rather than a method on an entity, because +judging values does not need one -- and needing one would mean an invalid +one had been built. It is annotated with the members it judges +(`Unpack[...]`), so the entity's dataclass fields, its configuration +TypedDict, its member table and its value routine are four spellings of +one set, and a test holds them to it. `unchecked` builds without asking, +for a caller that has already asked. The dataclass constructor asks, then +builds. `must_understand` moves to a class variable and out of the configuration entirely. It is a property of the *kind* of metadata -- a codec is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 6447b88a9b..dd7d25a926 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -14,7 +14,6 @@ ChunkGridEntity, Loc, MemberTypes, - ValueRoutine, is_integer, one_of, problem, @@ -202,44 +201,6 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: return frozenset(lengths) if len(lengths) != 0 else None -def _value_problems( - **members: Unpack[RectilinearChunkGridConfiguration], -) -> tuple[ValidationProblem, ...]: - """Every chunk extent, bare or run-length encoded, must be positive. - - A run's count must be positive too: a run of zero chunks is a way - of writing nothing at all, and the empty spelling already exists. - """ - found: list[ValidationProblem] = [] - for dim, spec in enumerate(members["chunk_shapes"]): - loc: tuple[str | int, ...] = ("chunk_shapes", dim) - if isinstance(spec, int): - if spec < 1: - found.extend( - problem(loc, f"expected a positive chunk extent, got {spec}", "invalid_value") - ) - continue - for position, item in enumerate(spec): - if isinstance(item, int): - if item < 1: - found.extend( - problem( - (*loc, position), - f"expected a positive chunk extent, got {item}", - "invalid_value", - ) - ) - elif item[0] < 1 or item[1] < 1: - found.extend( - problem( - (*loc, position), - f"expected a positive [size, count] pair, got {item!r}", - "invalid_value", - ) - ) - return tuple(found) - - @dataclass(frozen=True) class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" @@ -255,7 +216,45 @@ class RectilinearChunkGrid(ChunkGridEntity): "chunk_shapes": (True, _is_dim_specs), } - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[RectilinearChunkGridConfiguration], + ) -> tuple[ValidationProblem, ...]: + """Every chunk extent, bare or run-length encoded, must be positive. + + A run's count must be positive too: a run of zero chunks is a way + of writing nothing at all, and the empty spelling already exists. + """ + found: list[ValidationProblem] = [] + for dim, spec in enumerate(members["chunk_shapes"]): + loc: tuple[str | int, ...] = ("chunk_shapes", dim) + if isinstance(spec, int): + if spec < 1: + found.extend( + problem( + loc, f"expected a positive chunk extent, got {spec}", "invalid_value" + ) + ) + continue + for position, item in enumerate(spec): + if isinstance(item, int): + if item < 1: + found.extend( + problem( + (*loc, position), + f"expected a positive chunk extent, got {item}", + "invalid_value", + ) + ) + elif item[0] < 1 or item[1] < 1: + found.extend( + problem( + (*loc, position), + f"expected a positive [size, count] pair, got {item!r}", + "invalid_value", + ) + ) + return tuple(found) def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 45522fef1c..4a0faef9d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( ChunkGridEntity, MemberTypes, - ValueRoutine, is_int, problem, sequence_of, @@ -64,27 +63,6 @@ class RegularChunkGridObject(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[RegularChunkGridConfiguration], -) -> tuple[ValidationProblem, ...]: - """Every chunk extent must be at least one element. - - A chunk of zero elements along an axis covers nothing, so no - finite number of them tiles the axis; a negative one is - meaningless. Whether there is one extent *per array dimension* is - a question for the document, and the rules layer asks it. - """ - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(members["chunk_shape"]) - if extent < 1 - ) - - @dataclass(frozen=True) class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" @@ -96,7 +74,26 @@ class RegularChunkGrid(ChunkGridEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"chunk_shape": (True, sequence_of(is_int))} - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[RegularChunkGridConfiguration], + ) -> tuple[ValidationProblem, ...]: + """Every chunk extent must be at least one element. + + A chunk of zero elements along an axis covers nothing, so no + finite number of them tiles the axis; a negative one is + meaningless. Whether there is one extent *per array dimension* is + a question for the document, and the rules layer asks it. + """ + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(members["chunk_shape"]) + if extent < 1 + ) def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 6b3fd73f8d..05c1367c56 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -15,7 +15,6 @@ CodecEntity, CodecKind, MemberTypes, - ValueRoutine, is_int, one_of, problem, @@ -92,44 +91,6 @@ class BloscCodecObject(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[BloscCodecConfiguration], -) -> tuple[ValidationProblem, ...]: - """The value constraints the spec places on a blosc configuration.""" - found: list[ValidationProblem] = [] - clevel = members["clevel"] - if not 0 <= clevel <= 9: - found.extend( - problem(("clevel",), f"expected an integer in [0, 9], got {clevel}", "invalid_value") - ) - blocksize = members["blocksize"] - if blocksize < 0: - found.extend( - problem( - ("blocksize",), - f"expected a non-negative integer, got {blocksize}", - "invalid_value", - ) - ) - shuffle = members["shuffle"] - typesize = members.get("typesize") - # Only where it means something: under `noshuffle` the spec says - # "the value is ignored", and `canonical` drops it. - if typesize is not None and shuffle != BLOSC_NO_SHUFFLE and typesize < 1: - found.extend( - problem(("typesize",), f"expected a positive integer, got {typesize}", "invalid_value") - ) - if shuffle != BLOSC_NO_SHUFFLE and typesize is None: - found.extend( - problem( - ("typesize",), - f"typesize is required when shuffle is {shuffle!r}", - "missing_key", - ) - ) - return tuple(found) - - @dataclass(frozen=True) class BloscCodec(CodecEntity): """The `blosc` codec, coerced from its metadata. @@ -161,7 +122,47 @@ class BloscCodec(CodecEntity): "typesize": (False, is_int), } - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[BloscCodecConfiguration], + ) -> tuple[ValidationProblem, ...]: + """The value constraints the spec places on a blosc configuration.""" + found: list[ValidationProblem] = [] + clevel = members["clevel"] + if not 0 <= clevel <= 9: + found.extend( + problem( + ("clevel",), f"expected an integer in [0, 9], got {clevel}", "invalid_value" + ) + ) + blocksize = members["blocksize"] + if blocksize < 0: + found.extend( + problem( + ("blocksize",), + f"expected a non-negative integer, got {blocksize}", + "invalid_value", + ) + ) + shuffle = members["shuffle"] + typesize = members.get("typesize") + # Only where it means something: under `noshuffle` the spec says + # "the value is ignored", and `canonical` drops it. + if typesize is not None and shuffle != BLOSC_NO_SHUFFLE and typesize < 1: + found.extend( + problem( + ("typesize",), f"expected a positive integer, got {typesize}", "invalid_value" + ) + ) + if shuffle != BLOSC_NO_SHUFFLE and typesize is None: + found.extend( + problem( + ("typesize",), + f"typesize is required when shuffle is {shuffle!r}", + "missing_key", + ) + ) + return tuple(found) def canonical(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 1e55acb258..07d7a03501 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -14,7 +14,6 @@ CodecEntity, CodecKind, MemberTypes, - ValueRoutine, is_int, problem, ) @@ -69,16 +68,6 @@ class GzipCodecObject(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[GzipCodecConfiguration], -) -> tuple[ValidationProblem, ...]: - """gzip compression levels run 0 to 9.""" - level = members["level"] - if not 0 <= level <= 9: - return problem(("level",), f"expected an integer in [0, 9], got {level}", "invalid_value") - return () - - @dataclass(frozen=True) class GzipCodec(CodecEntity): """The `gzip` codec, coerced from its metadata.""" @@ -92,7 +81,17 @@ class GzipCodec(CodecEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"level": (True, is_int)} - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[GzipCodecConfiguration], + ) -> tuple[ValidationProblem, ...]: + """gzip compression levels run 0 to 9.""" + level = members["level"] + if not 0 <= level <= 9: + return problem( + ("level",), f"expected an integer in [0, 9], got {level}", "invalid_value" + ) + return () def to_json(self) -> GzipCodecObject: return cast("GzipCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 7a614f3c8d..f3bcfb8883 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -16,7 +16,6 @@ CodecEntity, CodecKind, MemberTypes, - ValueRoutine, is_json_value, problem, ) @@ -76,27 +75,6 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[ScaleOffsetCodecConfiguration], -) -> tuple[ValidationProblem, ...]: - """Each value is a scalar of the array's type, so neither is null. - - The registry says each is "JSON-encoded per the input array's - fill-value rules", and no data type admits `null` as a fill value. - Which scalar it should be needs the data type, so that part is the - document's question, not this codec's. - """ - # Each member named outright: a TypedDict indexed by a loop variable - # has no type, and the two are different members rather than two of - # a kind. - found: list[ValidationProblem] = [] - if members.get("offset", UNSET) is None: - found.extend(problem(("offset",), "expected a scalar, got null", "invalid_value")) - if members.get("scale", UNSET) is None: - found.extend(problem(("scale",), "expected a scalar, got null", "invalid_value")) - return tuple(found) - - @dataclass(frozen=True) class ScaleOffsetCodec(CodecEntity): """The `scale_offset` codec, coerced from its metadata. @@ -125,7 +103,26 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: """ return incoming - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[ScaleOffsetCodecConfiguration], + ) -> tuple[ValidationProblem, ...]: + """Each value is a scalar of the array's type, so neither is null. + + The registry says each is "JSON-encoded per the input array's + fill-value rules", and no data type admits `null` as a fill value. + Which scalar it should be needs the data type, so that part is the + document's question, not this codec's. + """ + # Each member named outright: a TypedDict indexed by a loop variable + # has no type, and the two are different members rather than two of + # a kind. + found: list[ValidationProblem] = [] + if members.get("offset", UNSET) is None: + found.extend(problem(("offset",), "expected a scalar, got null", "invalid_value")) + if members.get("scale", UNSET) is None: + found.extend(problem(("scale",), "expected a scalar, got null", "invalid_value")) + return tuple(found) def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 89ff8dff02..15a8b177c9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -18,7 +18,6 @@ Loc, MemberTypes, Opaque, - ValueRoutine, is_int, one_of, problem, @@ -153,25 +152,6 @@ class ShardingIndexedMembers(TypedDict): index_location: NotRequired[ShardingIndexLocation] -def _value_problems( - **members: Unpack[ShardingIndexedMembers], -) -> tuple[ValidationProblem, ...]: - """Every inner chunk extent must be at least one element. - - Nothing about the two pipelines: their codecs are entities, and an - entity exists only if its own values are allowed. - """ - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(members["chunk_shape"]) - if extent < 1 - ) - - @dataclass(frozen=True) class ShardingIndexedCodec(CodecEntity): """The `sharding_indexed` codec, coerced from its metadata. @@ -198,7 +178,24 @@ class ShardingIndexedCodec(CodecEntity): "index_location": (False, one_of(SHARDING_INDEX_LOCATION)), } - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[ShardingIndexedMembers], + ) -> tuple[ValidationProblem, ...]: + """Every inner chunk extent must be at least one element. + + Nothing about the two pipelines: their codecs are entities, and an + entity exists only if its own values are allowed. + """ + return tuple( + ValidationProblem( + ("chunk_shape", position), + f"expected a positive chunk extent, got {extent}", + "invalid_value", + ) + for position, extent in enumerate(members["chunk_shape"]) + if extent < 1 + ) @classmethod def prepare( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 5c178d31ac..a2d84bca49 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -14,7 +14,6 @@ CodecEntity, CodecKind, MemberTypes, - ValueRoutine, is_int, problem, sequence_of, @@ -66,24 +65,6 @@ class TransposeCodecObject(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[TransposeCodecConfiguration], -) -> tuple[ValidationProblem, ...]: - """`order` must permute its own axes. - - Whether it permutes the *array's* axes is a different question -- - it needs the array's rank -- and the rules layer asks that one. - """ - order = members["order"] - if sorted(order) != list(range(len(order))): - return problem( - ("order",), - f"expected a permutation of 0..{len(order) - 1}, got {order!r}", - "invalid_value", - ) - return () - - @dataclass(frozen=True) class TransposeCodec(CodecEntity): """The `transpose` codec, coerced from its metadata.""" @@ -96,7 +77,23 @@ class TransposeCodec(CodecEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"order": (True, sequence_of(is_int))} - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[TransposeCodecConfiguration], + ) -> tuple[ValidationProblem, ...]: + """`order` must permute its own axes. + + Whether it permutes the *array's* axes is a different question -- + it needs the array's rank -- and the rules layer asks that one. + """ + order = members["order"] + if sorted(order) != list(range(len(order))): + return problem( + ("order",), + f"expected a permutation of 0..{len(order) - 1}, got {order!r}", + "invalid_value", + ) + return () def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index b23487385a..79b95d6d0a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -17,7 +17,6 @@ CodecEntity, CodecKind, MemberTypes, - ValueRoutine, is_bool, is_int, problem, @@ -78,20 +77,6 @@ class ZstdCodecObject(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[ZstdCodecConfiguration], -) -> tuple[ValidationProblem, ...]: - """zstd compression levels run -131072 to 22.""" - level = members["level"] - if not ZSTD_MIN_LEVEL <= level <= ZSTD_MAX_LEVEL: - return problem( - ("level",), - f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {level}", - "invalid_value", - ) - return () - - @dataclass(frozen=True) class ZstdCodec(CodecEntity): """The `zstd` codec, coerced from its metadata.""" @@ -109,7 +94,19 @@ class ZstdCodec(CodecEntity): "checksum": (False, is_bool), } - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[ZstdCodecConfiguration], + ) -> tuple[ValidationProblem, ...]: + """zstd compression levels run -131072 to 22.""" + level = members["level"] + if not ZSTD_MIN_LEVEL <= level <= ZSTD_MAX_LEVEL: + return problem( + ("level",), + f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {level}", + "invalid_value", + ) + return () def to_json(self) -> ZstdCodecObject: return cast("ZstdCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8d74618302..a3a75fe59a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( MemberTypes, StorageClass, - ValueRoutine, is_int, one_of, problem, @@ -78,23 +77,6 @@ class NumpyDatetime64(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[NumpyDatetime64Configuration], -) -> tuple[ValidationProblem, ...]: - """`scale_factor` counts units per step, so it is positive. - - The upper bound is numpy's: the field is a signed 32-bit integer. - """ - scale_factor = members["scale_factor"] - if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - return problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", - "invalid_value", - ) - return () - - @dataclass(frozen=True) class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" @@ -111,7 +93,22 @@ class NumpyDatetime64DataType(NumpyTimeDataType): "scale_factor": (True, is_int), } - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[NumpyDatetime64Configuration], + ) -> tuple[ValidationProblem, ...]: + """`scale_factor` counts units per step, so it is positive. + + The upper bound is numpy's: the field is a signed 32-bit integer. + """ + scale_factor = members["scale_factor"] + if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + return problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", + "invalid_value", + ) + return () def to_json(self) -> NumpyDatetime64: return cast("NumpyDatetime64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index d70969a5af..e3caa35093 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( MemberTypes, StorageClass, - ValueRoutine, is_int, one_of, problem, @@ -98,23 +97,6 @@ class NumpyTimedelta64(TypedDict, closed=True): ] -def _value_problems( - **members: Unpack[NumpyTimedelta64Configuration], -) -> tuple[ValidationProblem, ...]: - """`scale_factor` counts units per step, so it is positive. - - The upper bound is numpy's: the field is a signed 32-bit integer. - """ - scale_factor = members["scale_factor"] - if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - return problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", - "invalid_value", - ) - return () - - @dataclass(frozen=True) class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" @@ -131,7 +113,22 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): "scale_factor": (True, is_int), } - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems( + **members: Unpack[NumpyTimedelta64Configuration], + ) -> tuple[ValidationProblem, ...]: + """`scale_factor` counts units per step, so it is positive. + + The upper bound is numpy's: the field is a signed 32-bit integer. + """ + scale_factor = members["scale_factor"] + if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + return problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", + "invalid_value", + ) + return () def to_json(self) -> NumpyTimedelta64: return cast("NumpyTimedelta64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index f1201f59ed..0168a8458b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -16,7 +16,6 @@ MemberTypes, Opaque, StorageClass, - ValueRoutine, problem, ) @@ -153,47 +152,6 @@ class StructMembers(TypedDict): fields: tuple[StructFieldComponent, ...] -def _value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ...]: - """What a struct can judge about its own fields. - - Names have to exist, be non-empty and be distinct, because a fill - value addresses fields by name. Field types have to be fixed-size, - because a record's layout is otherwise not determined. Nothing about - a field type's own values: it is an entity, so it exists only if - those are allowed. - """ - fields = members["fields"] - found: list[ValidationProblem] = [] - if len(fields) == 0: - found.extend(problem(("fields",), "expected at least one struct field", "invalid_value")) - seen: dict[str, int] = {} - for index, field in enumerate(fields): - at: Loc = ("fields", index) - if field.name == "": - found.extend(problem((*at, "name"), "expected a non-empty field name", "invalid_value")) - first = seen.setdefault(field.name, index) - if first != index: - found.extend( - problem( - (*at, "name"), - f"duplicate field name {field.name!r}, already used by field {first}", - "invalid_value", - ) - ) - if ( - isinstance(field.data_type, DataTypeEntity) - and field.data_type.storage_class() == "variable_length" - ): - found.extend( - problem( - (*at, "data_type"), - "struct fields must use fixed-size data types", - "invalid_value", - ) - ) - return tuple(found) - - @dataclass(frozen=True) class StructDataType(DataTypeEntity): """The `struct` data type, coerced from its metadata. @@ -211,7 +169,50 @@ class StructDataType(DataTypeEntity): configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = {"fields": (True, _is_fields)} - value_problems: ClassVar[ValueRoutine] = staticmethod(_value_problems) + @staticmethod + def value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ...]: + """What a struct can judge about its own fields. + + Names have to exist, be non-empty and be distinct, because a fill + value addresses fields by name. Field types have to be fixed-size, + because a record's layout is otherwise not determined. Nothing about + a field type's own values: it is an entity, so it exists only if + those are allowed. + """ + fields = members["fields"] + found: list[ValidationProblem] = [] + if len(fields) == 0: + found.extend( + problem(("fields",), "expected at least one struct field", "invalid_value") + ) + seen: dict[str, int] = {} + for index, field in enumerate(fields): + at: Loc = ("fields", index) + if field.name == "": + found.extend( + problem((*at, "name"), "expected a non-empty field name", "invalid_value") + ) + first = seen.setdefault(field.name, index) + if first != index: + found.extend( + problem( + (*at, "name"), + f"duplicate field name {field.name!r}, already used by field {first}", + "invalid_value", + ) + ) + if ( + isinstance(field.data_type, DataTypeEntity) + and field.data_type.storage_class() == "variable_length" + ): + found.extend( + problem( + (*at, "data_type"), + "struct fields must use fixed-size data types", + "invalid_value", + ) + ) + return tuple(found) @classmethod def prepare( diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 17a7d1e4b3..c48f5a32ad 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,7 +11,7 @@ import copy import dataclasses -from typing import get_type_hints +from typing import get_args, get_type_hints import pytest @@ -162,6 +162,27 @@ def test_a_required_member_rules_out_the_bare_spelling( assert entity.configuration_required == (required != 0) +@pytest.mark.parametrize( + ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) +) +def test_the_value_routine_takes_the_members_it_will_be_given( + entity: type[MetadataEntity], configuration: type | None +) -> None: + # `coerce` calls it as `value_problems(**members)`, which no type can + # check: members is a dict built at run time. So the fourth spelling + # of the set is checked here. A `struct` and a `sharding_indexed` + # annotate a TypedDict of their own rather than the configuration -- + # `prepare` has replaced field objects with entities by then -- but + # that changes the member *types*, never which members there are. + if entity.value_problems is MetadataEntity.value_problems: + return + # The annotation is `Unpack[X]`; X is what says which members. + (members,) = get_args(get_type_hints(entity.value_problems)["members"]) + fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} + assert set(get_type_hints(members)) == fields + assert configuration is not None, "a routine with no configuration to judge" + + def test_every_registered_entity_is_checked_here() -> None: registered = { f"{field}:{identifier}" From ac213b2ea37865e3536be0a27374343ad9dbeb82 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:31:44 +0200 Subject: [PATCH 052/107] build(zarr-metadata): typecheck on the current pyright The pin was on 1.1.404 because 1.1.405 regressed PEP 661 sentinel typing in class attributes (microsoft/pyright#11115), which this package's `UNSET` depends on. 1.1.414 reads it correctly again -- `int | UNSET` on the attribute, narrowing to `int` past an `is not UNSET` -- so the pin moves and the comment stops describing a bug that is fixed. One error surfaced with it, and it was a real one: a helper took `dict[str, object]` and handed it to `jsonschema`, which wants JSON. It takes `Mapping[str, JSONValue]`, which is what every caller passes. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- packages/zarr-metadata/justfile | 10 +++++----- packages/zarr-metadata/pyproject.toml | 5 ++--- .../zarr-metadata/tests/model/test_pydantic_module.py | 6 +++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index d9a0d2a5f6..5c0deaa9c5 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -13,12 +13,12 @@ test *args: lint: uvx ruff check . -# Pinned to the last pyright that types PEP 661 sentinels in class attributes -# correctly; 1.1.405+ regressed (microsoft/pyright#11115). Unpin when fixed. -pyright_version := "1.1.404" +# Pinned so a pyright release cannot turn CI red on its own schedule. Bump it +# deliberately: run the new version, read what it found, then move the pin. +pyright_version := "1.1.414" -# CI runs pyright on python 3.11; the pinned pyright predates 3.14, whose -# stdlib it cannot parse, so pin the interpreter to match CI. +# Run under the interpreter CI runs pyright under, so a version-dependent +# stdlib or stub difference shows up here rather than only in CI. # Type-check the package, sources and tests alike typecheck: uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index bfe7dce924..8bdf578cf4 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -125,9 +125,8 @@ checks = [ "PR06", ] -# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in -# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel -# relies on. Use the same pin locally; unpin when the fix lands. +# The pyright version lives in the justfile, which is what CI runs; pinning it +# keeps a pyright release from turning CI red on its own schedule. [tool.pyright] include = ["src", "tests"] # `tests` is a package imported as `tests.*`, which pytest resolves from the diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 410692a054..eaf870ea74 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -7,12 +7,14 @@ import json import warnings +from collections.abc import Mapping import pytest from jsonschema import Draft202012Validator from pydantic import BaseModel, TypeAdapter, ValidationError import zarr_metadata.pydantic as zmp +from zarr_metadata._common import JSONValue from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, @@ -156,7 +158,9 @@ def test_v2_recursive_structured_dtype_is_in_pydantic_schema() -> None: assert Draft202012Validator(adapter.json_schema()).is_valid(doc) -def _assert_runtime_and_schema_reject(field_type: object, document: dict[str, object]) -> None: +def _assert_runtime_and_schema_reject( + field_type: object, document: Mapping[str, JSONValue] +) -> None: adapter = TypeAdapter(field_type) with pytest.raises(ValidationError): adapter.validate_python(document) From 55608d7a51569c8ce1c8417119c57cc7bc473b17 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:35:40 +0200 Subject: [PATCH 053/107] feat(zarr-metadata): an entity is checked when it is declared Three more ways to write an entity that type-checks and then misbehaves somewhere that will not name it. A value rule in `__post_init__` never runs on a document: `coerce` builds through `unchecked`, which bypasses `__init__`. So the rule holds for a hand-built entity and is silently absent for every entity read from a file -- the one direction that matters. `RawBytesDataType` was doing exactly this, and had a second copy of the rule in its own `coerce` to cover the gap; it now states the rule once, in `value_problems`. A field shadowing an inherited class variable -- `must_understand`, `kind` -- goes into the configuration and into the JSON, while the class variable is what the rest of the layer reads. A class variable the entity owes and did not declare is now derived from the annotations rather than listed in `required_class_vars`. The list had `identifier`, `kind` and `scalar_storage` on it; `bounds`, `hex_parser`, `largest` and `component` were annotated on the data type families and never added, so an integer type missing its bounds raised `AttributeError` from whichever method reached it first. Judging values needs the entity's members, which are its fields, not its configuration keys: `r` keeps its width in its name and has no configuration at all. `_members` is the fields; serialization takes the configuration keys from `member_types` as before. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.10.md | 9 ++ .../src/zarr_metadata/v3/_entity.py | 115 +++++++++++++++--- .../src/zarr_metadata/v3/data_type/raw.py | 27 ++-- .../zarr-metadata/tests/v3/test_entities.py | 16 +-- .../tests/v3/test_extension_api.py | 41 +++++++ 5 files changed, 172 insertions(+), 36 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index bc9ba73ab4..5806acfb8d 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -32,3 +32,12 @@ this layer replaced, over 800 documents: 89 report fewer problems than before, and **no document changes verdict**. The report that survives is the one that has to be fixed first, and the JSON is still there on the `Opaque` standing in for the entity. + +Registering an entity is checked at class creation, because every way of +getting it wrong type-checks cleanly and then fails somewhere that will +not name the class: a class variable the entity owes and did not declare +(read off the annotations, so a family adding one cannot forget to +require it), a field shadowing one, a value rule in `__post_init__` -- +which `unchecked` never reaches, so it would hold for a hand-built entity +and be silently absent for every entity read from a document -- and a +member whose default contradicts whether the spec requires it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 8523c76176..6f0a03202a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -42,7 +42,16 @@ from copy import deepcopy from dataclasses import MISSING, Field, dataclass, fields from types import MappingProxyType -from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast +from typing import ( + TYPE_CHECKING, + ClassVar, + Final, + Literal, + TypeAlias, + TypeVar, + cast, + get_origin, +) from typing_extensions import TypeIs @@ -291,6 +300,35 @@ class Opaque: reason: Literal["out_of_scope", "invalid"] +def _is_class_var(annotation: object) -> bool: + """Whether an annotation says `ClassVar`. + + `from __future__ import annotations` leaves them as strings, so this + reads the text when it gets one -- the same thing `dataclasses` does, + and for the same reason: resolving the name needs a module namespace + that is not available while the class is still being built. + """ + if isinstance(annotation, str): + stripped = annotation.strip() + return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) + return get_origin(annotation) is ClassVar + + +def _declared_class_vars(cls: type) -> dict[str, type]: + """Every class variable annotated anywhere in `cls`'s ancestry. + + Mapped to the class that annotated it, so a message can say where the + requirement comes from. Base first, so a redeclaration names the + nearest ancestor. + """ + found: dict[str, type] = {} + for ancestor in reversed(cls.__mro__): + for name, annotation in vars(ancestor).get("__annotations__", {}).items(): + if _is_class_var(annotation): + found[name] = ancestor + return found + + # No `slots=True`, deliberately. It rebuilds the class, which on Python # 3.11 and 3.12 leaves the zero-argument `super()` *in that same class's # body* pointing at the class it replaced. Several entities call `super()` @@ -362,12 +400,12 @@ class MetadataEntity: """ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: - """Refuse a subclass that forgot to say what it is. + """Refuse a subclass that is not an entity this layer can use. - `identifier` and the per-kind class variables carry no default, - so a subclass omitting one type-checks cleanly and then raises - `AttributeError` from whichever method is reached first. Saying so - here makes it an import-time error in the extension's own module. + Every check here has the same shape: something that type-checks + cleanly and then goes wrong later, somewhere that will not name + this class. An import-time error in the extension's own module is + the one place the author is looking. `base=True` for a class that exists to add a class variable rather than to be an entity -- `CodecEntity`, `IntegerDataType`. @@ -384,9 +422,42 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: "`value_problems`, which takes the members rather than an entity" ) raise TypeError(msg) - missing = [name for name in cls.required_class_vars if not hasattr(cls, name)] + if "__post_init__" in cls.__dict__: + # `coerce` builds through `unchecked`, which bypasses + # `__init__` and so never reaches `__post_init__`. Rules put + # there would hold for a hand-built entity and be silently + # absent for every entity read from a document -- the one + # direction that matters. + msg = ( + f"{cls.__name__} defines `__post_init__`, which `unchecked` does " + "not reach; value rules belong in `value_problems`" + ) + raise TypeError(msg) + annotated = _declared_class_vars(cls) + shadowed = [ + name + for name in vars(cls).get("__annotations__", {}) + if name in annotated + and annotated[name] is not cls + and not _is_class_var(vars(cls)["__annotations__"][name]) + ] + if len(shadowed) != 0: + # A field of that name would go into `member_types`, into the + # configuration, and into the JSON -- while the class variable + # it shadows is what every other part of this layer reads. + msg = ( + f"{cls.__name__} declares {', '.join(shadowed)} as a field, " + "shadowing a class variable of the same name" + ) + raise TypeError(msg) + # A class variable annotated with no value anywhere in the + # ancestry is one the concrete entity owes: `identifier` for all + # of them, `kind` for a codec, `bounds` for an integer type. + # Derived rather than listed, so adding one to a family cannot + # forget to require it. + missing = [name for name in annotated if not hasattr(cls, name)] if len(missing) != 0: - msg = f"{cls.__name__} does not declare {', '.join(missing)}" + msg = f"{cls.__name__} does not declare {', '.join(sorted(missing))}" raise TypeError(msg) # A member's default decides whether the entity can exist without # it, so the two kinds have opposite rules. `@dataclass` has not @@ -436,9 +507,6 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: ) raise TypeError(msg) - required_class_vars: ClassVar[tuple[str, ...]] = ("identifier",) - """Every class variable a concrete entity of this kind must declare.""" - @classmethod def accepts(cls, name: str) -> bool: """Whether `name` denotes this entity. @@ -532,7 +600,7 @@ def configuration(self) -> dict[str, object]: the entity's own dict would let them mutate a frozen entity through the document it returned. """ - return deepcopy(self._members()) + return deepcopy(self._configuration_members()) value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) """Every value among the members the spec disallows. @@ -565,11 +633,24 @@ def __post_init__(self) -> None: raise MetadataValidationError(found) def _members(self) -> dict[str, object]: - """The configuration members present on this entity, unrendered. + """Every member this entity holds, unrendered. + + The dataclass's own fields, which is what `value_problems` + judges: a member is a member whether or not the JSON spells it + as a configuration key. The raw-bytes family is the case that + separates the two -- its width lives in its name, so it has a + field and no configuration at all. + """ + return { + field_.name: value + for field_ in fields(self) + if (value := getattr(self, field_.name)) is not UNSET + } + + def _configuration_members(self) -> dict[str, object]: + """The members a configuration object would spell out. - What `value_problems` and `configuration` are both built from; - `configuration` may render a member that is itself an entity, and - `value_problems` wants it as it is. + `_members` minus anything the envelope carries some other way. """ return { key: value @@ -643,7 +724,6 @@ class CodecEntity(MetadataEntity, base=True): """An entity that occupies a position in the codec pipeline.""" kind: ClassVar[CodecKind] - required_class_vars: ClassVar[tuple[str, ...]] = ("identifier", "kind") variable_size: ClassVar[bool] = False """Whether this codec's output size depends on the bytes it is given. @@ -710,7 +790,6 @@ class DataTypeEntity(MetadataEntity, base=True): """ scalar_storage: ClassVar[StorageClass] - required_class_vars: ClassVar[tuple[str, ...]] = ("identifier", "scalar_storage") def storage_class(self) -> StorageClass | None: """How one scalar occupies bytes, or None if undetermined. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index d9a718ccfd..e9ddab0541 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -12,7 +12,9 @@ from dataclasses import dataclass from typing import ClassVar, Final, NewType, Self, cast -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from typing_extensions import TypedDict, Unpack + +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( Coerced, @@ -97,6 +99,12 @@ def _name_problems(name: str) -> tuple[ValidationProblem, ...]: return () +class RawBytesMembers(TypedDict): + """A raw-bytes type's members: the spelling, which carries the width.""" + + data_type_name: str + + @dataclass(frozen=True) class RawBytesDataType(DataTypeEntity): """An `r` raw-bytes data type, coerced from its metadata. @@ -138,21 +146,20 @@ def coerce(cls, value: object, context: object) -> Coerced[Self]: # its fill values are still judged. Returning nothing here let # a stray key hide every other problem in the document. found = problem(("configuration",), "'r' takes no configuration", "unknown_key") - found = (*found, *_name_problems(name)) + found = (*found, *cls.value_problems(data_type_name=name)) if any(entry.kind != "unknown_key" for entry in found): return None, found return cls.unchecked(data_type_name=name), found - def __post_init__(self) -> None: - """This family's validity is in its name, not a configuration. + @staticmethod + def value_problems(**members: Unpack[RawBytesMembers]) -> tuple[ValidationProblem, ...]: + """This family's validity is in its name, not in a configuration. - So the base's member-driven check has nothing to look at, and - this one supplies it. + Which is the one place a member is not a configuration key, and + why `value_problems` judges the entity's fields rather than its + configuration: there is no configuration here to judge. """ - super().__post_init__() - found = _name_problems(self.data_type_name) - if len(found) != 0: - raise MetadataValidationError(found) + return _name_problems(members["data_type_name"]) def to_json(self) -> ZarrV3MetadataFieldJSON: return cast("ZarrV3MetadataFieldJSON", self.data_type_name) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index c48f5a32ad..b958334c36 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -163,24 +163,24 @@ def test_a_required_member_rules_out_the_bare_spelling( @pytest.mark.parametrize( - ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) + "entity", [entity for entity, _ in CONFIGURATIONS.values()], ids=list(CONFIGURATIONS) ) def test_the_value_routine_takes_the_members_it_will_be_given( - entity: type[MetadataEntity], configuration: type | None + entity: type[MetadataEntity], ) -> None: # `coerce` calls it as `value_problems(**members)`, which no type can - # check: members is a dict built at run time. So the fourth spelling - # of the set is checked here. A `struct` and a `sharding_indexed` - # annotate a TypedDict of their own rather than the configuration -- - # `prepare` has replaced field objects with entities by then -- but - # that changes the member *types*, never which members there are. + # check: members is a dict built at run time. So the correspondence + # is checked here, against the fields rather than the configuration + # -- `r` holds a member that is not a configuration key, and a + # `struct` and a `sharding_indexed` annotate a TypedDict of their own + # because `prepare` has replaced field objects with entities by then. + # Neither changes which members there are. if entity.value_problems is MetadataEntity.value_problems: return # The annotation is `Unpack[X]`; X is what says which members. (members,) = get_args(get_type_hints(entity.value_problems)["members"]) fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} assert set(get_type_hints(members)) == fields - assert configuration is not None, "a routine with no configuration to judge" def test_every_registered_entity_is_checked_here() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 3a7db5e810..9add9b1e9a 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -26,6 +26,7 @@ CodecKind, Context, DataTypeEntity, + IntegerDataType, MemberTypes, MetadataEntity, Opaque, @@ -275,3 +276,43 @@ class Stale(CodecEntity): # pyright: ignore[reportUnusedClass] def problems(self) -> tuple[ValidationProblem, ...]: return () + + +def test_error_an_entity_may_not_validate_in_post_init() -> None: + # `coerce` builds through `unchecked`, which never reaches + # `__post_init__`, so a rule there holds for a hand-built entity and + # is silently absent for every entity read from a document. + with pytest.raises(TypeError, match="`unchecked` does not reach"): + + @dataclass(frozen=True) + class Eager(CodecEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.eager" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def __post_init__(self) -> None: + raise AssertionError + + +def test_error_a_field_may_not_shadow_a_class_variable() -> None: + # A field of that name goes into the configuration and into the JSON, + # while the class variable it shadows is what the rest of the layer + # reads -- so the entity would claim one thing and behave as another. + with pytest.raises(TypeError, match="shadowing a class variable"): + + @dataclass(frozen=True) + class Negotiable(CodecEntity): # pyright: ignore[reportUnusedClass] + must_understand: bool = True # pyright: ignore[reportIncompatibleVariableOverride] + + identifier: ClassVar[str] = "acme.negotiable" + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: + # `bounds` is annotated on `IntegerDataType` and bound by none of it, + # so every concrete integer type owes one. Nothing lists it: the + # requirement is read off the annotation. + with pytest.raises(TypeError, match="does not declare bounds"): + + @dataclass(frozen=True) + class Int24DataType(IntegerDataType): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.int24" From 4746900ef945ee42d53b137499c92b83fead77ba Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:38:52 +0200 Subject: [PATCH 054/107] refactor(zarr-metadata): an entity claims its own names A family covers many names with one class, and `canonical_name` was how one got found: a package-level function that folded every `r` spelling onto the family's key. Correct for the one family this package models, and a wall for anyone else -- a third party could write `accepts` and have it never consulted, because no name would fold onto their key. `resolve` now tries the name as a key, and on a miss asks each entity in that point's table whether the name is one of its own. Registering a family is registering an entity; no table of spellings is left in the package. The common case is still a lookup, and the scan only runs for a name nothing is keyed by, where the alternative was returning nothing. `_extension_points` loses the function that was its reason to exist and keeps the constants. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.9.md | 7 ++ .../src/zarr_metadata/v3/_extension_points.py | 26 +++----- .../src/zarr_metadata/v3/_registry.py | 35 ++++++---- .../tests/v3/test_extension_api.py | 58 ++++++++++++++++- .../tests/v3/test_extension_points.py | 54 ++++++++-------- .../tests/v3/test_shape_properties.py | 64 ++++++++++--------- 6 files changed, 159 insertions(+), 85 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.9.md b/packages/zarr-metadata/changes/4379.feature.9.md index 16bf298c17..c30ce8422c 100644 --- a/packages/zarr-metadata/changes/4379.feature.9.md +++ b/packages/zarr-metadata/changes/4379.feature.9.md @@ -19,3 +19,10 @@ exhaustive two-case union that narrows. `Context.coerce` is overloaded on the extension point, so it returns the entity type for that point rather than the base, and the extension-point constants keep their `Literal` types so a call written with one of them gets the narrow result. + +A *family* -- one class covering a parameterized set of names, as the +raw-byte types cover every `r` -- is registered like anything else. +It takes an invented identifier no document can write and claims its own +names through `accepts`, which `resolve` asks when no key matches. There +is no table of spellings anywhere in the package, so a third party can +register a family without changing it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py index 0ed9ac77e0..eea1b6542d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py @@ -1,13 +1,14 @@ -"""The Zarr v3 extension points, and how names are keyed under them. +"""The Zarr v3 extension points. Names are unique only within an extension point (`bytes` is both a core codec and a registered data type), so every table in this package is -keyed by `(field, canonical name)`. +keyed by `(field, name)`. -`canonical_name` is identity except for raw-byte data types: every `r` -spelling, valid or not, maps to `RAW_BYTES_FAMILY`, so a malformed member -of that family is reported as a misspelling rather than passing as an -unknown extension. Canonical names are lookup keys and are never emitted. +A name that no key matches may still belong to a *family* -- one class +covering many spellings, as the raw-byte data types cover every `r`. +`Context.resolve` asks each entity through `accepts`, so a family is +registered like anything else and this module holds no table of +spellings. """ from __future__ import annotations @@ -17,17 +18,10 @@ CHUNK_KEY_ENCODING, CODECS, DATA_TYPE, + STORAGE_TRANSFORMERS, ExtensionPointField, ) -from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY, RAW_BYTES_NAME_PATTERN - - -def canonical_name(field: ExtensionPointField, name: str) -> str: - """`name` reduced to the key this package tables it under.""" - if field == DATA_TYPE and RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None: - return RAW_BYTES_FAMILY - return name - +from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY __all__ = [ "CHUNK_GRID", @@ -35,6 +29,6 @@ def canonical_name(field: ExtensionPointField, name: str) -> str: "CODECS", "DATA_TYPE", "RAW_BYTES_FAMILY", + "STORAGE_TRANSFORMERS", "ExtensionPointField", - "canonical_name", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 24e2d67f0f..33e57c5d33 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -12,10 +12,11 @@ `zarr-extensions` registers and this package models. A name in neither is not rejected — extension openness — it is simply not judged. -Identifiers are the `name` the metadata carries, with one exception. Every -`r` spelling is one data-type family, so the family registers under an -invented identifier that no real name can collide with; `canonical_name` -folds a spelling onto it. +Identifiers are the `name` the metadata carries, with one exception. A +family covers many names with one class -- every `r` spelling is one +data-type family -- so it registers under an invented identifier that no +real name can collide with, and recognizes its own names through +`accepts`. """ from __future__ import annotations @@ -43,7 +44,6 @@ CHUNK_KEY_ENCODING, CODECS, DATA_TYPE, - canonical_name, ) from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid @@ -250,16 +250,23 @@ def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] Out of scope is not an error: an unknown name may be an extension this reader does not model, and openness means leaving it unjudged. - The entity has the last word, via `accepts`. Folding is what finds - a candidate -- every `r` spelling is tabled under one invented - identifier -- and the candidate is what says whether the name is - really one of its own. Otherwise the identifier itself would be a - name a document could write. + The entity has the last word, via `accepts`. A name that is a key + still has to be claimed, because a family's key is an invented + identifier that no document may write; and a name that is not a + key may still belong to a family, which is what the scan is for. + A third party registers one the same way, with no table of + spellings anywhere in this package. """ - entity = self.tables()[field].get(canonical_name(field, name)) - if entity is None or not entity.accepts(name): - return None - return entity + table = self.tables()[field] + entity = table.get(name) + if entity is not None: + return entity if entity.accepts(name) else None + # A family covers many names with one class, so its entry cannot + # be keyed by all of them; it is keyed by an invented identifier + # and recognizes its own. Asked only when the name is not a key, + # so the common case stays a lookup. First match wins, and two + # entities claiming one name is a scope that contradicts itself. + return next((candidate for candidate in table.values() if candidate.accepts(name)), None) @overload def coerce( diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 9add9b1e9a..a92d0c75fc 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -6,8 +6,9 @@ from __future__ import annotations +import re from dataclasses import dataclass -from typing import ClassVar +from typing import TYPE_CHECKING, ClassVar, Self, cast import pytest @@ -24,6 +25,7 @@ ChunkGridEntity, CodecEntity, CodecKind, + Coerced, Context, DataTypeEntity, IntegerDataType, @@ -32,12 +34,17 @@ Opaque, StorageClass, is_int, + named_configuration, problem, ) ACME_MAX_ACCELERATION = 65537 +if TYPE_CHECKING: + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + + @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): """A third-party compressor.""" @@ -316,3 +323,52 @@ def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: @dataclass(frozen=True) class Int24DataType(IntegerDataType): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.int24" + + +# A third-party *family*: one class covering a parameterized set of names, +# the way `r` covers every raw-byte width. +ACME_FIXED_PATTERN = re.compile(r"acme\.fixed(\d+)") + + +@dataclass(frozen=True) +class AcmeFixedDataType(DataTypeEntity): + """`acme.fixedN`, a fixed-width type for every N.""" + + data_type_name: str + + identifier: ClassVar[str] = "acme.fixed" + scalar_storage: ClassVar[StorageClass] = "multi_byte" + + @classmethod + def accepts(cls, name: str) -> bool: + return ACME_FIXED_PATTERN.fullmatch(name) is not None + + @classmethod + def coerce(cls, value: object, context: object) -> Coerced[Self]: + name, _, _ = named_configuration(value) + if name is None or not cls.accepts(name): + return None, problem((), "expected an 'acme.fixedN' data type") + return cls.unchecked(data_type_name=name), () + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return cast("ZarrV3MetadataFieldJSON", self.data_type_name) + + +def test_a_third_party_can_register_a_family() -> None: + # One class for an unbounded set of names. Nothing in the package + # holds a table of spellings: the entity registers under an invented + # identifier and `resolve` asks it, so a family is registered exactly + # like a single name. + scope = CORE_AND_EXTENSIONS.extended_with( + data_type={AcmeFixedDataType.identifier: AcmeFixedDataType} + ) + for name in ("acme.fixed8", "acme.fixed128"): + assert scope.resolve("data_type", name) is AcmeFixedDataType + entity, problems = scope.coerce("data_type", name) + assert problems == () + assert isinstance(entity, AcmeFixedDataType) + assert entity.to_json() == name + # The invented identifier is not a name a document may write, and a + # near-miss is still nobody's. + assert scope.resolve("data_type", AcmeFixedDataType.identifier) is None + assert scope.resolve("data_type", "acme.fixed") is None diff --git a/packages/zarr-metadata/tests/v3/test_extension_points.py b/packages/zarr-metadata/tests/v3/test_extension_points.py index edbfea6560..b3355f205a 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_points.py +++ b/packages/zarr-metadata/tests/v3/test_extension_points.py @@ -1,4 +1,4 @@ -"""Tests for extension-point name canonicalization.""" +"""Tests for how a name reaches the entity that answers for it.""" from __future__ import annotations @@ -9,33 +9,38 @@ CODECS, DATA_TYPE, RAW_BYTES_FAMILY, - canonical_name, ) +from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.data_type.numpy_datetime64 import NumpyDatetime64DataType +from zarr_metadata.v3.data_type.raw import RawBytesDataType +from zarr_metadata.v3.data_type.uint8 import Uint8DataType +from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS, MetadataEntity -# (field, name, expected canonical key) — identity everywhere except the -# parameterized raw-bytes family. -CANONICAL_CASES: dict[str, tuple[str, str, str]] = { - "plain-dtype": (DATA_TYPE, "uint8", "uint8"), - "dotted-dtype": (DATA_TYPE, "numpy.datetime64", "numpy.datetime64"), - "raw-8": (DATA_TYPE, "r8", RAW_BYTES_FAMILY), - "raw-24": (DATA_TYPE, "r24", RAW_BYTES_FAMILY), - # Malformed members canonicalize into the family too: a misspelling of +# (field, name, the entity that answers for it — None when nothing does) +RESOLUTIONS: dict[str, tuple[str, str, type[MetadataEntity] | None]] = { + "plain-dtype": (DATA_TYPE, "uint8", Uint8DataType), + "dotted-dtype": (DATA_TYPE, "numpy.datetime64", NumpyDatetime64DataType), + "raw-8": (DATA_TYPE, "r8", RawBytesDataType), + "raw-24": (DATA_TYPE, "r24", RawBytesDataType), + # A malformed member reaches the family too: a misspelling of # something we model must be reported as such, not pass as an unknown # third-party extension. - "raw-not-multiple-of-8": (DATA_TYPE, "r12", RAW_BYTES_FAMILY), - "raw-zero": (DATA_TYPE, "r0", RAW_BYTES_FAMILY), - # Canonicalization is field-aware: the r family is a data type. - "raw-shaped-codec-name": (CODECS, "r8", "r8"), - "codec": (CODECS, "blosc", "blosc"), - "unknown": (CODECS, "zfpy", "zfpy"), + "raw-not-multiple-of-8": (DATA_TYPE, "r12", RawBytesDataType), + "raw-zero": (DATA_TYPE, "r0", RawBytesDataType), + # Tables are per point, so the family cannot be reached from another. + "raw-shaped-codec-name": (CODECS, "r8", None), + "codec": (CODECS, "blosc", BloscCodec), + "unknown": (CODECS, "zfpy", None), + # The family's key is invented, so no document may write it. + "the-family-key-itself": (DATA_TYPE, RAW_BYTES_FAMILY, None), } -@pytest.mark.parametrize( - ("field", "name", "expected"), CANONICAL_CASES.values(), ids=list(CANONICAL_CASES) -) -def test_canonical_name(field: str, name: str, expected: str) -> None: - assert canonical_name(field, name) == expected # type: ignore[arg-type] +@pytest.mark.parametrize(("field", "name", "expected"), RESOLUTIONS.values(), ids=list(RESOLUTIONS)) +def test_a_name_resolves_to_the_entity_that_answers_for_it( + field: str, name: str, expected: type[MetadataEntity] | None +) -> None: + assert CORE_AND_EXTENSIONS.resolve(field, name) is expected # type: ignore[arg-type] def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: @@ -60,10 +65,9 @@ def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: def test_forging_the_family_sentinel_cannot_change_a_verdict() -> None: - # A literal "r" data type mislabels nothing: the rules layer matches - # the family through the name pattern, not through the table key, so - # no validation verdict depends on the sentinel being unforgeable. - assert canonical_name(DATA_TYPE, RAW_BYTES_FAMILY) == RAW_BYTES_FAMILY + # A literal "r" data type mislabels nothing: the family claims its + # names through `accepts`, not through the table key, so no validation + # verdict depends on the sentinel being unforgeable. document = { "zarr_format": 3, "node_type": "array", diff --git a/packages/zarr-metadata/tests/v3/test_shape_properties.py b/packages/zarr-metadata/tests/v3/test_shape_properties.py index 1bcf8a3bf2..6140d92d72 100644 --- a/packages/zarr-metadata/tests/v3/test_shape_properties.py +++ b/packages/zarr-metadata/tests/v3/test_shape_properties.py @@ -1,16 +1,15 @@ -"""Generative invariants for raw-bytes name canonicalization. - -Scoped to canonicalization deliberately. Two earlier tests here asserted -that a shape verdict exists exactly when `(field, canonical_name(...))` -is in `modelled_entities()` — but both sides were computed from -`_ENTITY_SHAPES` through the same call, so they restated the lookup -rather than testing it, and could not fail. Worse, they could not catch -the bug class they named (a lookup passing the wrong field), because both -sides used the same field. `tests/rules/test_registry.py` covers that -with real assertions. - -Canonicalization is a genuine fit for generative testing: the family is -unbounded, so an example-based test can only sample it. +"""Generative invariants for how a family is resolved. + +Scoped to resolution deliberately. Two earlier tests here asserted that a +shape verdict exists exactly when `(field, canonical_name(...))` is in +`modelled_entities()` — but both sides were computed from `_ENTITY_SHAPES` +through the same call, so they restated the lookup rather than testing it, +and could not fail. Worse, they could not catch the bug class they named +(a lookup passing the wrong field), because both sides used the same +field. `tests/rules/test_registry.py` covers that with real assertions. + +A family is a genuine fit for generative testing: it is unbounded, so an +example-based test can only sample it. """ from __future__ import annotations @@ -18,34 +17,41 @@ from hypothesis import given from hypothesis import strategies as st -from zarr_metadata.v3._extension_points import ( +from zarr_metadata.v3.data_type.raw import RawBytesDataType +from zarr_metadata.v3.entity import ( CHUNK_GRID, CODECS, + CORE_AND_EXTENSIONS, DATA_TYPE, - RAW_BYTES_FAMILY, - canonical_name, ) @given(width=st.integers(min_value=0, max_value=2**32)) -def test_every_numeric_r_spelling_folds_to_one_key(width: int) -> None: +def test_every_numeric_r_spelling_resolves_to_the_family(width: int) -> None: # Including malformed widths (0, 12, anything not a multiple of 8): - # canonicalization is by grammar shape, not validity, so a misspelled - # member of a family we model is reported as a misspelling rather than - # passing as an unknown third-party extension. - assert canonical_name(DATA_TYPE, f"r{width}") == RAW_BYTES_FAMILY + # the family claims a name by grammar shape, not by validity, so a + # misspelled member of a family we model is reported as a misspelling + # rather than passing as an unknown third-party extension. + assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, f"r{width}") is RawBytesDataType @given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from([CODECS, CHUNK_GRID])) -def test_r_shaped_names_are_identity_outside_data_types(width: int, field: str) -> None: +def test_r_shaped_names_resolve_to_nothing_outside_data_types(width: int, field: str) -> None: # The family belongs to `data_type`; a codec that happens to be named - # `r8` must not be folded into it. - name = f"r{width}" - assert canonical_name(field, name) == name # type: ignore[arg-type] + # `r8` must not reach it. + assert CORE_AND_EXTENSIONS.resolve(field, f"r{width}") is None # type: ignore[arg-type] -@given( - name=st.text(min_size=1).filter(lambda s: not (s.startswith("r") and s[1:].isdigit())), +# The scan `resolve` falls back to asks every entity, so a name no entity +# claims has to come back as nothing however many are registered. +_UNCLAIMED = st.text(min_size=1).filter( + lambda name: ( + not (name.startswith("r") and name[1:].isdigit()) + and name not in CORE_AND_EXTENSIONS.entities["data_type"] + ) ) -def test_non_family_names_are_identity(name: str) -> None: - assert canonical_name(DATA_TYPE, name) == name + + +@given(name=_UNCLAIMED) +def test_a_name_no_entity_claims_resolves_to_nothing(name: str) -> None: + assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, name) is None From 945f33c82f06a4317e3f58110b89b09eca38ca9a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:40:15 +0200 Subject: [PATCH 055/107] docs(zarr-metadata): to_json is faithful to the members it models `to_json` promised that a document read and written comes back as it went in, which is true of every member an entity models and not of one it does not. An unknown configuration member is reported as `unknown_key` and not held, so writing back drops it. That is survivable because of what the survivability is for: the key is reported rather than fatal so it cannot hide every other finding about its entity, and `from_json` -- the reader that does not hand back problems -- refuses the document outright. Reaching the drop means taking the problems as data and going on past that one. Both halves are now tests rather than a claim in a docstring. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.6.md | 5 ++ .../src/zarr_metadata/v3/_entity.py | 13 +++-- .../zarr-metadata/tests/v3/test_entities.py | 47 ++++++++++++++++++- 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.6.md b/packages/zarr-metadata/changes/4379.feature.6.md index e55a5e8476..d17f261545 100644 --- a/packages/zarr-metadata/changes/4379.feature.6.md +++ b/packages/zarr-metadata/changes/4379.feature.6.md @@ -18,3 +18,8 @@ its entity. Model round-trips preserve unmodeled members. Shape-exact `TypeIs` guards still reject them because the corresponding TypedDicts are closed. + +An unknown member is reported, not held, so an entity written back does +not carry it. Only a caller who took the problems as data and continued +past that one can reach that: `from_json` raises on it, and the JSON the +caller passed in is still the JSON it passed in. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 6f0a03202a..dcdc65ff9b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -694,9 +694,16 @@ def unchecked(cls, **members: object) -> Self: def to_json(self) -> ZarrV3MetadataFieldJSON: """This entity as a document would write it. - Faithful to every member: read a document, write it back, and the - members come out as they went in. Ask `canonical` first if you - want the simplest equivalent spelling. + Faithful to every member it models: read a document, write it + back, and those come out as they went in. Ask `canonical` first + if you want the simplest equivalent spelling. + + A member this entity does not model is not one of them. It is + reported as `unknown_key` and not held, so writing back drops it + -- which only a caller who took the problems as data and went on + past that one can reach, because `from_json` raises on it. A + caller who needs the bytes preserved has the JSON it passed in, + and `Opaque` is where unmodelled metadata belongs. What is *not* preserved is the envelope's spelling, because the entity does not model it: a bare name, `{"name": x}`, and diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index b958334c36..df26aec092 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,10 +11,11 @@ import copy import dataclasses -from typing import get_args, get_type_hints +from typing import Any, cast, get_args, get_type_hints import pytest +from zarr_metadata.model import MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( @@ -68,7 +69,7 @@ from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType -from zarr_metadata.v3.entity import MetadataEntity +from zarr_metadata.v3.entity import ArrayDocumentV3, MetadataEntity # Each registered entity, paired with the TypedDict its constructor # mirrors. Keyed by `:`, because an identifier is only @@ -547,3 +548,45 @@ def test_to_json_shares_no_mutable_state_with_the_entity(field: str, written: ob else: configuration[key] = "clobbered" assert entity.to_json() == baseline + + +def test_a_member_the_entity_does_not_model_is_not_written_back() -> None: + # `unknown_key` is survivable so that one stray member cannot hide + # every other finding about its entity -- a concession about + # reporting, not a promise to carry the member. The entity holds what + # it models, so writing back drops it. + entry = { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "typesize": 2, + "blocksize": 0, + "typo_key": 1, + }, + } + codec, problems = CORE_AND_EXTENSIONS.coerce("codecs", entry) + assert [(p.loc, p.kind) for p in problems] == [(("configuration", "typo_key"), "unknown_key")] + assert isinstance(codec, MetadataEntity) + assert "typo_key" not in codec.to_json()["configuration"] # type: ignore[index,operator] + + +def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: + # Which is why dropping it is survivable: the reader that does not + # hand back problems does not hand back the entity either. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes", {"name": "gzip", "configuration": {"level": 1, "typo_key": 1}}), + } + with pytest.raises(MetadataValidationError) as caught: + ArrayDocumentV3.from_json(cast("Any", document)) + assert (("codecs", 1, "configuration", "typo_key"), "unknown_key") in { + (problem.loc, problem.kind) for problem in caught.value.problems + } From 87e4f4be4b41ade044072237d0b2a4b0f8fb6a90 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 22:43:30 +0200 Subject: [PATCH 056/107] docs(zarr-metadata): regenerate the differential against the current tree The numbers in the changelog were measured before this branch refused `must_understand: false` below the top level, and one of them had since become false: two documents do change verdict. Both go from valid to invalid, and both are that refusal. Re-run against the same pre-refactor tree over 40,000 documents through all four entry points: nothing this layer accepts was rejected before, every document valid under both reports identically, and among those invalid under both, 4,397 report fewer problems while 15,259 report more -- the second figure being the one the old text had no way to mention, since a problem that used to stand down the rest of an entity no longer does. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.10.md | 17 ++++++++++++----- packages/zarr-metadata/changes/4379.misc.2.md | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index 5806acfb8d..84ad4a2597 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -27,11 +27,18 @@ really can skip. The cost is reporting density on documents that are already invalid. An entity that cannot be built cannot be asked anything, so a grid with one bad extent no longer pins its good axes and a shard with one bad inner -extent no longer judges its pipelines. Measured against the rule registry -this layer replaced, over 800 documents: 89 report fewer problems than -before, and **no document changes verdict**. The report that survives is -the one that has to be fixed first, and the JSON is still there on the -`Opaque` standing in for the entity. +extent no longer judges its pipelines. The report that survives is the one +that has to be fixed first, and the JSON is still there on the `Opaque` +standing in for the entity. + +Measured against the rule registry this layer replaced, over a shared +corpus of 40,000 documents through all four `validate_*` entry points: +**nothing this layer accepts was rejected before**. Two verdicts change, +both the other way and both the `must_understand: false` refusal above -- +one at the top level, one nested in a shard's pipelines. Every document +valid under both reports identically. Among those invalid under both, +4,397 report fewer problems and 15,259 report more, the latter because +a problem that used to stand down the rest of an entity no longer does. Registering an entity is checked at class creation, because every way of getting it wrong type-checks cleanly and then fails somewhere that will diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 4e8d38e9a3..564c6da470 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -18,7 +18,7 @@ Two results worth recording. Eleven `zarr.json` documents written by this repository's own `create_array` -- float32+blosc, sharded, string, datetime64, complex128, structured, transposed -- validate clean. And against the rule registry this layer replaced, over one shared corpus of -800 documents, no verdict differs in the laxer direction and the valid +40,000 documents, no verdict differs in the laxer direction and the valid documents report identically. One thing deliberately not done. An entity's `to_json` returns its own From 35e5ede724df7bf731a06f459f8a0d5320f72cf4 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sun, 20 Sep 2026 23:12:31 +0200 Subject: [PATCH 057/107] refactor(zarr-metadata): a configuration says its own member types `member_types` restated the configuration TypedDict in a form the program could run: the same keys, the same requiredness, and a check per member that the annotation already implied. Two spellings of one fact, held together by a test. An entity now names its `configuration_type` and the table is read off it. `int`, `bool`, `str`, a `Literal`, a homogeneous tuple and the JSON-value alias each imply their check, which covers 24 of the 30 members; the 6 that remain are exactly those whose annotation names another structure, and they keep a hand-written entry. `configuration_required` is derived as well. Three guards, because the configuration is now the only place these may be said: restating `configuration_required`, giving a hand-written entry a requiredness the configuration does not, and leaving a member with no check from either source. Requiredness comes from the resolved annotation, not `__required_keys__` -- a TypedDict computes that from unresolved strings and reports every member required under `from __future__ import annotations`, which would have made an extension author's optional members silently mandatory. A derived `Literal` check sorts the values it reports, because two `Literal`s over one value-set compare and hash equal, so the order `get_args` gives is whichever was built first in the process. No behaviour change: over the same 40,000-document corpus, no verdict, problem location or problem kind differs, and the only message change is the order of the permitted values a `one_of` lists. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 --- .../zarr-metadata/changes/4379.feature.7.md | 21 +++ .../src/zarr_metadata/v3/_entity.py | 154 +++++++++++++++- .../v3/chunk_grid/rectilinear.py | 4 +- .../zarr_metadata/v3/chunk_grid/regular.py | 7 +- .../v3/chunk_key_encoding/default.py | 7 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 7 +- .../src/zarr_metadata/v3/codec/blosc.py | 13 +- .../src/zarr_metadata/v3/codec/bytes.py | 5 +- .../src/zarr_metadata/v3/codec/cast_value.py | 5 +- .../src/zarr_metadata/v3/codec/gzip.py | 6 +- .../zarr_metadata/v3/codec/scale_offset.py | 8 +- .../v3/codec/sharding_indexed.py | 7 +- .../src/zarr_metadata/v3/codec/transpose.py | 7 +- .../src/zarr_metadata/v3/codec/zstd.py | 10 +- .../v3/data_type/numpy_datetime64.py | 11 +- .../v3/data_type/numpy_timedelta64.py | 10 +- .../src/zarr_metadata/v3/data_type/struct.py | 6 +- .../src/zarr_metadata/v3/entity.py | 32 +++- .../zarr-metadata/tests/v3/test_entities.py | 166 +++++++----------- .../tests/v3/test_extension_api.py | 71 +++++++- 20 files changed, 345 insertions(+), 212 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index f33f0e1e02..25a8bf9982 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -26,3 +26,24 @@ type check falls back to absent, because a bad `index_location` says nothing about whether a shard's pipelines are well formed and silencing them would lose a real judgment. A *required* one stops the entity, because there is no honest reading of a `blosc` whose level is a string. + +The type judgment is read off the configuration TypedDict rather than +written twice. An entity names its `configuration_type`, and which +members exist, which may be left out, and how each is type-checked all +follow from it -- `int`, `bool`, `str`, a `Literal`, a homogeneous tuple +and the JSON-value alias each imply their check. Six members across the +package keep a hand-written entry, all of them ones whose annotation +names another structure (a shard's two pipelines, a `cast_value` target +and its scalar map, a rectilinear grid's chunk shapes, a struct's +fields), where reading the check off the annotation would take a +TypedDict-to-checker compiler. `configuration_required` follows too, +since the spec ties the bare-name spelling to whether any member is +required. + +Three things a class may no longer restate, because the configuration +already says them: `configuration_required`, the requiredness of a +hand-written entry, and a member with no check at all. Requiredness is +taken from the resolved annotation rather than `__required_keys__`, +which a TypedDict computes from unresolved strings and gets wrong under +`from __future__ import annotations` -- an extension author's optional +members would otherwise become silently mandatory. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index dcdc65ff9b..0a4abc9644 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -47,14 +47,20 @@ ClassVar, Final, Literal, + NotRequired, + Protocol, + Required, TypeAlias, TypeVar, cast, + get_args, get_origin, + get_type_hints, ) -from typing_extensions import TypeIs +from typing_extensions import ReadOnly, TypeIs +from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( MetadataValidationError, @@ -267,6 +273,95 @@ def coerce_members( return members, tuple(problems), frozenset(unreadable) +class ConfigurationType(Protocol): + """What this layer reads off a configuration TypedDict. + + A Protocol rather than `type`, so that what is read off it is stated + rather than assumed. Deliberately not `__required_keys__`: under + `from __future__ import annotations` a TypedDict computes that from + unresolved strings and reports every member required, which would + make an extension's optional members silently mandatory. Requiredness + is read from the resolved annotation instead. + """ + + __name__: str + + +def _unwrap(annotation: object) -> object: + """An annotation without the qualifiers that are not its type. + + `NotRequired` and `Required` say whether a member must be present, + which is the other half of a member table entry; `ReadOnly` says + nothing about the value at all. + """ + while get_origin(annotation) in (NotRequired, Required, ReadOnly): + (annotation,) = get_args(annotation) + return annotation + + +def check_for(annotation: object) -> TypeCheck | None: + """The check an annotation implies, or None if it implies none. + + None for an annotation naming another structure -- a nested + TypedDict, a recursive JSON alias, a tuple of either. Reading those + off the annotation would be a TypedDict-to-checker compiler, which + is a different package; the entity declares those itself. + """ + annotation = _unwrap(annotation) + if annotation is int: + return is_int + if annotation is bool: + return is_bool + if annotation is str: + return is_str + if annotation is JSONValue: + return is_json_value + if get_origin(annotation) is Literal: + # Sorted, because the order `get_args` reports is not the order + # the `Literal` was written in: two `Literal`s over the same + # values compare and hash equal, so the first one built anywhere + # in the process is the one every later one resolves to. The + # check is a membership test either way; this is so the message + # listing the values does not depend on import order. + return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(annotation))))) + if get_origin(annotation) is tuple: + arguments = get_args(annotation) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = check_for(arguments[0]) + return None if element is None else sequence_of(element) + return None + + +def is_required(annotation: object) -> bool: + """Whether a configuration member must be present. + + From the resolved annotation rather than the TypedDict's + `__required_keys__`, which is computed from unresolved strings and + is wrong for a module using `from __future__ import annotations`. + `ReadOnly` may wrap either way round, so it is peeled first. + """ + while get_origin(annotation) is ReadOnly: + (annotation,) = get_args(annotation) + return get_origin(annotation) is not NotRequired + + +def derive_member_types( + configuration: ConfigurationType, +) -> dict[str, tuple[bool, TypeCheck]]: + """The member table a configuration TypedDict already describes. + + Requiredness is the TypedDict's, and so is the check wherever the + annotation implies one. A member it does not imply one for is left + out, for the entity to declare. + """ + derived: dict[str, tuple[bool, TypeCheck]] = {} + for member, annotation in get_type_hints(configuration, include_extras=True).items(): + check = check_for(annotation) + if check is not None: + derived[member] = (is_required(annotation), check) + return derived + + ValueRoutine: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" """An entity's value-space judgment, over the members it was given.""" @@ -384,19 +479,31 @@ class MetadataEntity: an invented identifier that no real name can collide with. """ + configuration_type: ClassVar[ConfigurationType | None] = None + """The TypedDict describing this entity's `configuration` in JSON. + + None for an entity that has no configuration. Everything else about + the members is read off it at class creation, so the JSON shape is + stated once: `member_types` and `configuration_required` are both + derived, and the constructor is held to the same keys by + `tests/v3/test_entities.py`. + """ + member_types: ClassVar[MemberTypes] = MappingProxyType({}) """The configuration members, and the type each one takes. - The same keys as the configuration TypedDict, which is the same as the - constructor signature; `tests/v3/test_entities.py` holds the three - together. + Derived from `configuration_type`. A class declares an entry here + only for a member whose annotation names another structure -- a + nested TypedDict, a recursive JSON alias -- which is where reading + the check off the annotation would take a compiler. """ configuration_required: ClassVar[bool] = False """Whether the bare-name spelling says too little for this entity. The spec permits a bare name "if no configuration metadata is - required", so this is true exactly when some member is required. + required", so this is true exactly when some member is required -- + which the configuration TypedDict already says. """ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: @@ -433,6 +540,43 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: "not reach; value rules belong in `value_problems`" ) raise TypeError(msg) + if "configuration_required" in vars(cls): + msg = ( + f"{cls.__name__} declares `configuration_required`, which follows " + "from whether its configuration has a required member" + ) + raise TypeError(msg) + if cls.configuration_type is not None: + # Before every guard below, because they read the table. + declared = dict(vars(cls).get("member_types", {})) + derived = derive_member_types(cls.configuration_type) + undeclared = sorted( + set(get_type_hints(cls.configuration_type)) - set(derived) - set(declared) + ) + if len(undeclared) != 0: + msg = ( + f"{cls.__name__} declares no check for {', '.join(undeclared)}, " + "whose annotation does not imply one" + ) + raise TypeError(msg) + hints = get_type_hints(cls.configuration_type, include_extras=True) + misstated = sorted( + member + for member, (required, _) in declared.items() + if member in hints and required != is_required(hints[member]) + ) + if len(misstated) != 0: + # The check is the entity's to write; whether the member + # may be left out is the configuration's to say, and a + # declared entry that disagrees is the drift this + # derivation exists to rule out. + msg = ( + f"{cls.__name__} declares {', '.join(misstated)} with a requiredness " + "its configuration does not give it" + ) + raise TypeError(msg) + cls.member_types = {**derived, **declared} + cls.configuration_required = any(required for required, _ in cls.member_types.values()) annotated = _declared_class_vars(cls) shadowed = [ name diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index dd7d25a926..4eb945f0d8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -15,7 +15,6 @@ Loc, MemberTypes, is_integer, - one_of, problem, ) from zarr_metadata.v3._parts import ChunkGrid @@ -209,10 +208,9 @@ class RectilinearChunkGrid(ChunkGridEntity): chunk_shapes: tuple[RectilinearDimSpec, ...] identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME + configuration_type = RectilinearChunkGridConfiguration - configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = { - "kind": (True, one_of(RECTILINEAR_CHUNK_GRID_KIND)), "chunk_shapes": (True, _is_dim_specs), } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 4a0faef9d4..f521339032 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -12,10 +12,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, - MemberTypes, - is_int, problem, - sequence_of, ) from zarr_metadata.v3._parts import ChunkGrid @@ -70,9 +67,7 @@ class RegularChunkGrid(ChunkGridEntity): chunk_shape: tuple[int, ...] identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME - - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = {"chunk_shape": (True, sequence_of(is_int))} + configuration_type = RegularChunkGridConfiguration @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 20ff86dbbd..59e6f3688b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -14,9 +14,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( - MemberTypes, MetadataEntity, - one_of, ) DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" @@ -79,10 +77,7 @@ class DefaultChunkKeyEncoding(MetadataEntity): separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME - - member_types: ClassVar[MemberTypes] = { - "separator": (False, one_of(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)) - } + configuration_type = DefaultChunkKeyEncodingConfiguration def to_json(self) -> DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName: return cast( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index fb417d1ad8..337946a9d6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -20,9 +20,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( - MemberTypes, MetadataEntity, - one_of, ) V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" @@ -85,10 +83,7 @@ class V2ChunkKeyEncoding(MetadataEntity): separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME - - member_types: ClassVar[MemberTypes] = { - "separator": (False, one_of(V2_CHUNK_KEY_ENCODING_SEPARATOR)) - } + configuration_type = V2ChunkKeyEncodingConfiguration def to_json(self) -> V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName: return cast("V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 05c1367c56..2ba444496a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -14,9 +14,6 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - MemberTypes, - is_int, - one_of, problem, ) @@ -107,20 +104,12 @@ class BloscCodec(CodecEntity): typesize: int | UNSET = UNSET identifier: ClassVar[str] = BLOSC_CODEC_NAME + configuration_type = BloscCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" # Every member is required but `typesize`, which only means something # when shuffling; `problems` is where that conditional lives. - configuration_required: ClassVar[bool] = True - - member_types: ClassVar[MemberTypes] = { - "cname": (True, one_of(BLOSC_CNAME)), - "clevel": (True, is_int), - "shuffle": (True, one_of(BLOSC_SHUFFLE)), - "blocksize": (True, is_int), - "typesize": (False, is_int), - } @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 983b7a0dd5..2552415ad6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -15,8 +15,6 @@ CodecEntity, CodecKind, DataTypeEntity, - MemberTypes, - one_of, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -92,10 +90,9 @@ class BytesCodec(CodecEntity): endian: Endianness | UNSET = UNSET identifier: ClassVar[str] = BYTES_CODEC_NAME + configuration_type = BytesCodecConfiguration kind: ClassVar[CodecKind] = "array_bytes" - member_types: ClassVar[MemberTypes] = {"endian": (False, one_of(ENDIANNESS))} - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """The data type reaching here must have a raw byte representation. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 341c638c5b..f290ff53e9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -19,7 +19,6 @@ MemberTypes, Opaque, is_json_value, - one_of, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -188,13 +187,11 @@ class CastValueCodec(CodecEntity): scalar_map: ScalarMap | UNSET = UNSET identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME + configuration_type = CastValueCodecConfiguration kind: ClassVar[CodecKind] = "array_array" - configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = { "data_type": (True, _is_data_type_field), - "rounding": (False, one_of(CAST_ROUNDING_MODE)), - "out_of_range": (False, one_of(CAST_OUT_OF_RANGE_MODE)), "scalar_map": (False, _is_scalar_map), } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 07d7a03501..7f45ce6af8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -13,8 +13,6 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - MemberTypes, - is_int, problem, ) @@ -75,12 +73,10 @@ class GzipCodec(CodecEntity): level: int identifier: ClassVar[str] = GZIP_CODEC_NAME + configuration_type = GzipCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = {"level": (True, is_int)} - @staticmethod def value_problems( **members: Unpack[GzipCodecConfiguration], diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index f3bcfb8883..4292c89ab1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -15,8 +15,6 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - MemberTypes, - is_json_value, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -88,13 +86,9 @@ class ScaleOffsetCodec(CodecEntity): scale: JSONValue | UNSET = UNSET identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME + configuration_type = ScaleOffsetCodecConfiguration kind: ClassVar[CodecKind] = "array_array" - member_types: ClassVar[MemberTypes] = { - "offset": (False, is_json_value), - "scale": (False, is_json_value), - } - def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same array, element for element. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 15a8b177c9..898c4e487a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -18,10 +18,7 @@ Loc, MemberTypes, Opaque, - is_int, - one_of, problem, - sequence_of, ) from zarr_metadata.v3._parts import ( UNKNOWN_GRID, @@ -167,15 +164,13 @@ class ShardingIndexedCodec(CodecEntity): index_location: ShardingIndexLocation | UNSET = UNSET identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME + configuration_type = ShardingIndexedCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "array_bytes" - configuration_required: ClassVar[bool] = True member_types: ClassVar[MemberTypes] = { - "chunk_shape": (True, sequence_of(is_int)), "codecs": (True, _is_field_tuple), "index_codecs": (True, _is_field_tuple), - "index_location": (False, one_of(SHARDING_INDEX_LOCATION)), } @staticmethod diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index a2d84bca49..670be8c237 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -13,10 +13,7 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - MemberTypes, - is_int, problem, - sequence_of, ) from zarr_metadata.v3._parts import ArrayParts @@ -72,11 +69,9 @@ class TransposeCodec(CodecEntity): order: tuple[int, ...] identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME + configuration_type = TransposeCodecConfiguration kind: ClassVar[CodecKind] = "array_array" - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = {"order": (True, sequence_of(is_int))} - @staticmethod def value_problems( **members: Unpack[TransposeCodecConfiguration], diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 79b95d6d0a..a354289690 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -16,9 +16,6 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - MemberTypes, - is_bool, - is_int, problem, ) @@ -85,15 +82,10 @@ class ZstdCodec(CodecEntity): checksum: bool | UNSET = UNSET identifier: ClassVar[str] = ZSTD_CODEC_NAME + configuration_type = ZstdCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = { - "level": (True, is_int), - "checksum": (False, is_bool), - } - @staticmethod def value_problems( **members: Unpack[ZstdCodecConfiguration], diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index a3a75fe59a..d2cd548f38 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -11,16 +11,12 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( - MemberTypes, StorageClass, - is_int, - one_of, problem, ) from zarr_metadata.v3.data_type._families import NumpyTimeDataType from zarr_metadata.v3.data_type.numpy_timedelta64 import ( NUMPY_TIME_MAX_SCALE_FACTOR, - NUMPY_TIME_UNIT, ) NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" @@ -86,12 +82,7 @@ class NumpyDatetime64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME - - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = { - "unit": (True, one_of(NUMPY_TIME_UNIT)), - "scale_factor": (True, is_int), - } + configuration_type = NumpyDatetime64Configuration @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index e3caa35093..abf835c2bf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -11,10 +11,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( - MemberTypes, StorageClass, - is_int, - one_of, problem, ) from zarr_metadata.v3.data_type._families import NumpyTimeDataType @@ -106,12 +103,7 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME - - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = { - "unit": (True, one_of(NUMPY_TIME_UNIT)), - "scale_factor": (True, is_int), - } + configuration_type = NumpyTimedelta64Configuration @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 0168a8458b..4c10e9ff44 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -164,10 +164,12 @@ class StructDataType(DataTypeEntity): fields: tuple[StructFieldComponent, ...] identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME + configuration_type = StructConfiguration scalar_storage: ClassVar[StorageClass] = "single_byte" - configuration_required: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = {"fields": (True, _is_fields)} + member_types: ClassVar[MemberTypes] = { + "fields": (True, _is_fields), + } @staticmethod def value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index fc8b648ecf..399e9fe97f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -24,9 +24,12 @@ else: codec.json, codec.reason # 'out_of_scope': resolve it yourself -**Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, -`ChunkGridEntity` or `MetadataEntity`, declare `identifier` and -`member_types`, and put it in a `Context`: +**Writing an extension.** Describe the JSON with a TypedDict, subclass +`CodecEntity`, `DataTypeEntity`, `ChunkGridEntity` or `MetadataEntity`, +point at the TypedDict, and add it to a scope: + + class AcmeLz4Configuration(TypedDict, closed=True): + acceleration: NotRequired[int] @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): @@ -36,14 +39,29 @@ class AcmeLz4Codec(CodecEntity): identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" - member_types: ClassVar[MemberTypes] = {"acceleration": (False, is_int)} + configuration_type = AcmeLz4Configuration - SCOPE = Context({**CORE_AND_EXTENSIONS.entities, - "codecs": {**CORE_AND_EXTENSIONS.entities["codecs"], - AcmeLz4Codec.identifier: AcmeLz4Codec}}) + SCOPE = CORE_AND_EXTENSIONS.extended_with( + codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}, + ) validate_array_metadata_v3(document, context=SCOPE) +`configuration_type` is the only place the JSON shape is written. Which +members exist, which may be left out, and how each one is type-checked +are all read off it -- `member_types` is for the exception, a member +whose annotation names another structure. Value rules go in a +`value_problems` staticmethod annotated with the same TypedDict, which +runs only once every member has the type it declared: + + @staticmethod + def value_problems( + **members: Unpack[AcmeLz4Configuration], + ) -> tuple[ValidationProblem, ...]: + if "acceleration" not in members: + return () + ... + A name in no scope is not rejected -- that is what extension openness means -- so registering yours is how you get it judged rather than waved through. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index df26aec092..a8b4d4fa3a 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -20,29 +20,25 @@ from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, - RectilinearChunkGridConfiguration, ) -from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid, RegularChunkGridConfiguration +from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid from zarr_metadata.v3.chunk_key_encoding.default import ( DefaultChunkKeyEncoding, - DefaultChunkKeyEncodingConfiguration, ) from zarr_metadata.v3.chunk_key_encoding.v2 import ( V2ChunkKeyEncoding, - V2ChunkKeyEncodingConfiguration, ) -from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration -from zarr_metadata.v3.codec.bytes import BytesCodec, BytesCodecConfiguration -from zarr_metadata.v3.codec.cast_value import CastValueCodec, CastValueCodecConfiguration -from zarr_metadata.v3.codec.crc32c import Crc32cCodec, Empty -from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecConfiguration -from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec, ScaleOffsetCodecConfiguration +from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.codec.bytes import BytesCodec +from zarr_metadata.v3.codec.cast_value import CastValueCodec +from zarr_metadata.v3.codec.crc32c import Crc32cCodec +from zarr_metadata.v3.codec.gzip import GzipCodec +from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec from zarr_metadata.v3.codec.sharding_indexed import ( ShardingIndexedCodec, - ShardingIndexedCodecConfiguration, ) -from zarr_metadata.v3.codec.transpose import TransposeCodec, TransposeCodecConfiguration -from zarr_metadata.v3.codec.zstd import ZstdCodec, ZstdCodecConfiguration +from zarr_metadata.v3.codec.transpose import TransposeCodec +from zarr_metadata.v3.codec.zstd import ZstdCodec from zarr_metadata.v3.data_type.bool import BoolDataType from zarr_metadata.v3.data_type.bytes import BytesDataType from zarr_metadata.v3.data_type.complex64 import Complex64DataType @@ -55,16 +51,14 @@ from zarr_metadata.v3.data_type.int32 import Int32DataType from zarr_metadata.v3.data_type.int64 import Int64DataType from zarr_metadata.v3.data_type.numpy_datetime64 import ( - NumpyDatetime64Configuration, NumpyDatetime64DataType, ) from zarr_metadata.v3.data_type.numpy_timedelta64 import ( - NumpyTimedelta64Configuration, NumpyTimedelta64DataType, ) from zarr_metadata.v3.data_type.raw import RawBytesDataType from zarr_metadata.v3.data_type.string import StringDataType -from zarr_metadata.v3.data_type.struct import StructConfiguration, StructDataType +from zarr_metadata.v3.data_type.struct import StructDataType from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType @@ -75,97 +69,69 @@ # mirrors. Keyed by `:`, because an identifier is only # unique within its extension point -- `bytes` is both a codec and a data # type. -CONFIGURATIONS: dict[str, tuple[type[MetadataEntity], type | None]] = { - "codecs:blosc": (BloscCodec, BloscCodecConfiguration), - "codecs:bytes": (BytesCodec, BytesCodecConfiguration), - "codecs:cast_value": (CastValueCodec, CastValueCodecConfiguration), - "codecs:crc32c": (Crc32cCodec, Empty), - "codecs:gzip": (GzipCodec, GzipCodecConfiguration), - "codecs:scale_offset": (ScaleOffsetCodec, ScaleOffsetCodecConfiguration), - "codecs:sharding_indexed": (ShardingIndexedCodec, ShardingIndexedCodecConfiguration), - "codecs:transpose": (TransposeCodec, TransposeCodecConfiguration), - "codecs:zstd": (ZstdCodec, ZstdCodecConfiguration), - "chunk_grid:regular": (RegularChunkGrid, RegularChunkGridConfiguration), - "chunk_grid:rectilinear": (RectilinearChunkGrid, RectilinearChunkGridConfiguration), - "chunk_key_encoding:default": (DefaultChunkKeyEncoding, DefaultChunkKeyEncodingConfiguration), - "chunk_key_encoding:v2": (V2ChunkKeyEncoding, V2ChunkKeyEncodingConfiguration), - "data_type:numpy.datetime64": (NumpyDatetime64DataType, NumpyDatetime64Configuration), - "data_type:numpy.timedelta64": (NumpyTimedelta64DataType, NumpyTimedelta64Configuration), - # Bare entities: the name says everything, so there is no - # configuration TypedDict for the constructor to mirror. - "data_type:bool": (BoolDataType, None), - "data_type:int8": (Int8DataType, None), - "data_type:int16": (Int16DataType, None), - "data_type:int32": (Int32DataType, None), - "data_type:int64": (Int64DataType, None), - "data_type:uint8": (Uint8DataType, None), - "data_type:uint16": (Uint16DataType, None), - "data_type:uint32": (Uint32DataType, None), - "data_type:uint64": (Uint64DataType, None), - "data_type:float16": (Float16DataType, None), - "data_type:float32": (Float32DataType, None), - "data_type:float64": (Float64DataType, None), - "data_type:complex64": (Complex64DataType, None), - "data_type:complex128": (Complex128DataType, None), - "data_type:bytes": (BytesDataType, None), - "data_type:struct": (StructDataType, StructConfiguration), - "data_type:string": (StringDataType, None), - # The one exception. `r` is a family, so the class holds the - # spelling that picks a member of it -- a field with no configuration - # member behind it, because the name carries the information. - "data_type:r": (RawBytesDataType, None), +# Every registered entity, keyed by `:` -- an identifier +# is unique only within its extension point, and `bytes` is both a codec +# and a data type. What each one's configuration is comes off the class: +# `configuration_type` is the only place that says so, and the fields are +# held to it below. +ENTITIES: dict[str, type[MetadataEntity]] = { + "codecs:blosc": BloscCodec, + "codecs:bytes": BytesCodec, + "codecs:cast_value": CastValueCodec, + "codecs:crc32c": Crc32cCodec, + "codecs:gzip": GzipCodec, + "codecs:scale_offset": ScaleOffsetCodec, + "codecs:sharding_indexed": ShardingIndexedCodec, + "codecs:transpose": TransposeCodec, + "codecs:zstd": ZstdCodec, + "chunk_grid:regular": RegularChunkGrid, + "chunk_grid:rectilinear": RectilinearChunkGrid, + "chunk_key_encoding:default": DefaultChunkKeyEncoding, + "chunk_key_encoding:v2": V2ChunkKeyEncoding, + "data_type:numpy.datetime64": NumpyDatetime64DataType, + "data_type:numpy.timedelta64": NumpyTimedelta64DataType, + "data_type:bool": BoolDataType, + "data_type:int8": Int8DataType, + "data_type:int16": Int16DataType, + "data_type:int32": Int32DataType, + "data_type:int64": Int64DataType, + "data_type:uint8": Uint8DataType, + "data_type:uint16": Uint16DataType, + "data_type:uint32": Uint32DataType, + "data_type:uint64": Uint64DataType, + "data_type:float16": Float16DataType, + "data_type:float32": Float32DataType, + "data_type:float64": Float64DataType, + "data_type:complex64": Complex64DataType, + "data_type:complex128": Complex128DataType, + "data_type:bytes": BytesDataType, + "data_type:struct": StructDataType, + "data_type:string": StringDataType, + "data_type:r": RawBytesDataType, } -@pytest.mark.parametrize( - ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) -) -def test_the_constructor_mirrors_the_configuration( - entity: type[MetadataEntity], configuration: type | None -) -> None: - # `must_understand` belongs to the object, not the configuration, so it - # is the one field the two deliberately do not share. - if configuration is None: - expected = {"data_type_name"} if entity is RawBytesDataType else set() - assert {field.name for field in dataclasses.fields(entity)} - { - "must_understand" - } == expected - return +@pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) +def test_the_constructor_mirrors_the_configuration(entity: type[MetadataEntity]) -> None: + # The one correspondence still written by hand, and so the one that + # can still drift: the member table and `configuration_required` are + # now read off `configuration_type`, but the dataclass fields are + # not. It is also what catches an entity pointing at the wrong + # TypedDict, since the fields would stop matching. + # + # `must_understand` belongs to the object, not the configuration, so + # it is the one field the two deliberately do not share. + configuration = entity.configuration_type fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} - assert fields == set(get_type_hints(configuration)) - - -@pytest.mark.parametrize( - ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) -) -def test_the_member_table_mirrors_the_configuration( - entity: type[MetadataEntity], configuration: type | None -) -> None: - # The third spelling of the same set. Which members are *required* is - # in the TypedDict too, so that cannot drift either. if configuration is None: - assert entity.member_types == {} + # `r` keeps its width in its name, so it holds a member that + # is not a configuration key. + assert fields == ({"data_type_name"} if entity is RawBytesDataType else set()) return - assert set(entity.member_types) == set(get_type_hints(configuration)) - required = {key for key, (needed, _) in entity.member_types.items() if needed} - assert required == set(configuration.__required_keys__) # type: ignore[attr-defined] - - -@pytest.mark.parametrize( - ("entity", "configuration"), CONFIGURATIONS.values(), ids=list(CONFIGURATIONS) -) -def test_a_required_member_rules_out_the_bare_spelling( - entity: type[MetadataEntity], configuration: type | None -) -> None: - # The spec permits a bare name only "if no configuration metadata is - # required", so one flag follows from the other. - required = 0 if configuration is None else len(configuration.__required_keys__) # type: ignore[attr-defined] - assert entity.configuration_required == (required != 0) + assert fields == set(get_type_hints(configuration)) -@pytest.mark.parametrize( - "entity", [entity for entity, _ in CONFIGURATIONS.values()], ids=list(CONFIGURATIONS) -) +@pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) def test_the_value_routine_takes_the_members_it_will_be_given( entity: type[MetadataEntity], ) -> None: @@ -190,7 +156,7 @@ def test_every_registered_entity_is_checked_here() -> None: for field, entities in CORE_AND_EXTENSIONS.tables().items() for identifier in entities } - assert registered == set(CONFIGURATIONS) + assert registered == set(ENTITIES) def test_core_is_a_subset_of_core_and_extensions() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index a92d0c75fc..5a84a82571 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,9 +8,10 @@ import re from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Self, cast +from typing import TYPE_CHECKING, ClassVar, NotRequired, Self, cast import pytest +from typing_extensions import TypedDict, Unpack from zarr_metadata.model import UNSET, MetadataValidationError, ValidationProblem from zarr_metadata.rules import ( @@ -45,6 +46,12 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +class AcmeLz4Configuration(TypedDict, closed=True): + """The JSON shape of an `acme.lz4` configuration.""" + + acceleration: NotRequired[int] + + @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): """A third-party compressor.""" @@ -54,13 +61,18 @@ class AcmeLz4Codec(CodecEntity): identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" variable_size: ClassVar[bool] = True - member_types: ClassVar[MemberTypes] = {"acceleration": (False, is_int)} + configuration_type = AcmeLz4Configuration @staticmethod - def value_problems(**members: object) -> tuple[ValidationProblem, ...]: - acceleration = members.get("acceleration", UNSET) - if acceleration is UNSET or not isinstance(acceleration, int): + def value_problems( + **members: Unpack[AcmeLz4Configuration], + ) -> tuple[ValidationProblem, ...]: + # No defensive narrowing: a member that failed its type check + # never reaches here, so asking whether it is present is enough + # to have an int. + if "acceleration" not in members: return () + acceleration = members["acceleration"] if not 1 <= acceleration <= ACME_MAX_ACCELERATION: return problem( ("acceleration",), @@ -372,3 +384,52 @@ def test_a_third_party_can_register_a_family() -> None: # near-miss is still nobody's. assert scope.resolve("data_type", AcmeFixedDataType.identifier) is None assert scope.resolve("data_type", "acme.fixed") is None + + +def test_error_requiredness_may_not_be_restated() -> None: + # It is the configuration's to say. A declared entry exists for the + # check, which the annotation does not imply; saying the member is + # required as well is the drift the derivation removes. + with pytest.raises(TypeError, match="requiredness its configuration does not give it"): + + @dataclass(frozen=True) + class Insistent(CodecEntity): # pyright: ignore[reportUnusedClass] + acceleration: int | UNSET = UNSET + + identifier: ClassVar[str] = "acme.insistent" + kind: ClassVar[CodecKind] = "bytes_bytes" + configuration_type = AcmeLz4Configuration + member_types: ClassVar[MemberTypes] = {"acceleration": (True, is_int)} + + +def test_error_a_member_needs_a_check_from_somewhere() -> None: + # An annotation naming another structure implies no check, so the + # entity owes one. Silently skipping the member would let anything + # through where the TypedDict promised a shape. + class Nested(TypedDict, closed=True): + inner: AcmeLz4Configuration + + with pytest.raises(TypeError, match="declares no check for inner"): + + @dataclass(frozen=True) + class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] + inner: object + + identifier: ClassVar[str] = "acme.structured" + kind: ClassVar[CodecKind] = "bytes_bytes" + configuration_type = Nested + + +def test_error_a_bare_name_rule_may_not_be_restated() -> None: + # Whether the bare spelling is legal follows from whether any member + # is required, which the configuration already says. + with pytest.raises(TypeError, match="declares `configuration_required`"): + + @dataclass(frozen=True) + class Opinionated(CodecEntity): # pyright: ignore[reportUnusedClass] + acceleration: int | UNSET = UNSET + + identifier: ClassVar[str] = "acme.opinionated" + kind: ClassVar[CodecKind] = "bytes_bytes" + configuration_type = AcmeLz4Configuration + configuration_required: ClassVar[bool] = True From fa24b7167404a71d7025fce508c27ae55add6f19 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 09:15:23 +0200 Subject: [PATCH 058/107] refactor(zarr-metadata): an entity's fields are the schema The member table was read off a parallel TypedDict named by `configuration_type`, which restated what the dataclass fields already say: `level: int` is a required integer, `typesize: int | UNSET = UNSET` an optional one, `codecs: tuple[CodecEntity | Opaque, ...]` a sequence of nested metadata fields. The pointer and the restatement both go; the table is compiled from the fields. `check_for` grows from six branches into a small compiler over the shapes this package's metadata takes -- unions, fixed and homogeneous arrays, nested objects described by a TypedDict or a record dataclass, and nested metadata fields -- which covers every member modelled here. The six hand-written checks and all six `member_types` overrides are deleted with it. `Annotated[str, FROM_NAME]` marks the one field the envelope's name carries rather than a configuration key. Field annotations are resolved per class, skipping class variables by text, so a `ClassVar` naming something imported only for the type checker cannot fail class creation -- which `get_type_hints` on the base did, every time, because its string aliases named `Mapping` and `Callable` through `TYPE_CHECKING`. Those are runtime imports now. Over 40,000 documents: no verdict changes, no parse flips. One diagnostic moved: a malformed `[value, count]` pair in a rectilinear grid is reported at the offending element inside the pair, where the hand-written check reported at the pair. The sixteen entity modules lose 156 lines; the base gains 245, most of it the compiler and its docstrings. That is the trade: the mechanism is in one place, and what an extension author writes is the fields. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 44 +- .../src/zarr_metadata/v3/_entity.py | 477 +++++++++++++----- .../v3/chunk_grid/rectilinear.py | 44 -- .../zarr_metadata/v3/chunk_grid/regular.py | 1 - .../v3/chunk_key_encoding/default.py | 1 - .../zarr_metadata/v3/chunk_key_encoding/v2.py | 1 - .../src/zarr_metadata/v3/codec/blosc.py | 1 - .../src/zarr_metadata/v3/codec/bytes.py | 1 - .../src/zarr_metadata/v3/codec/cast_value.py | 48 -- .../src/zarr_metadata/v3/codec/gzip.py | 1 - .../zarr_metadata/v3/codec/scale_offset.py | 1 - .../v3/codec/sharding_indexed.py | 21 - .../src/zarr_metadata/v3/codec/transpose.py | 1 - .../src/zarr_metadata/v3/codec/zstd.py | 1 - .../v3/data_type/numpy_datetime64.py | 1 - .../v3/data_type/numpy_timedelta64.py | 1 - .../src/zarr_metadata/v3/data_type/raw.py | 6 +- .../src/zarr_metadata/v3/data_type/struct.py | 34 -- .../src/zarr_metadata/v3/entity.py | 31 +- .../zarr-metadata/tests/v3/test_entities.py | 77 ++- .../tests/v3/test_extension_api.py | 30 +- 21 files changed, 470 insertions(+), 353 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 25a8bf9982..880a24bf97 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -27,23 +27,29 @@ nothing about whether a shard's pipelines are well formed and silencing them would lose a real judgment. A *required* one stops the entity, because there is no honest reading of a `blosc` whose level is a string. -The type judgment is read off the configuration TypedDict rather than -written twice. An entity names its `configuration_type`, and which -members exist, which may be left out, and how each is type-checked all -follow from it -- `int`, `bool`, `str`, a `Literal`, a homogeneous tuple -and the JSON-value alias each imply their check. Six members across the -package keep a hand-written entry, all of them ones whose annotation -names another structure (a shard's two pipelines, a `cast_value` target -and its scalar map, a rectilinear grid's chunk shapes, a struct's -fields), where reading the check off the annotation would take a -TypedDict-to-checker compiler. `configuration_required` follows too, -since the spec ties the bare-name spelling to whether any member is -required. +The type judgment is read off the entity's own fields rather than +written twice. Which members exist, which may be left out (the type +admits `UNSET`), and how each is type-checked all follow from the field +annotations: `int`, `bool`, `str`, a `Literal` of names, the JSON-value +alias, an array homogeneous or fixed, a union of those, a nested object +described by a TypedDict or a record dataclass, and a nested metadata +field -- an entity type, with or without `Opaque`. That covers every +member this package models; `member_types` remains only for an +annotation the compiler does not read, and `Annotated[str, FROM_NAME]` +marks the one field carried by the envelope's name rather than a +configuration key (`r`). `configuration_required` follows too, since +the spec ties the bare-name spelling to whether any member is required. +The public JSON TypedDict is no longer read by the package at all: the +fields are held to its keys by a test, which is the one correspondence +still written by hand. -Three things a class may no longer restate, because the configuration -already says them: `configuration_required`, the requiredness of a -hand-written entry, and a member with no check at all. Requiredness is -taken from the resolved annotation rather than `__required_keys__`, -which a TypedDict computes from unresolved strings and gets wrong under -`from __future__ import annotations` -- an extension author's optional -members would otherwise become silently mandatory. +Three things a class may no longer restate, because the fields already +say them: `configuration_required`, the requiredness of a hand-written +entry, and a member with no check from either source. Field annotations +are resolved per class, skipping class variables by text, so a +`ClassVar` naming something imported only for the type checker cannot +fail class creation. + +One diagnostic became more precise: a malformed `[value, count]` pair in +a rectilinear grid is now reported at the offending element inside the +pair rather than at the pair. No verdict changes. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 0a4abc9644..d9a2f83f97 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -30,35 +30,42 @@ codec holds two codec pipelines, and neither can coerce its own configuration without knowing what names are in scope inside it. -The shared plumbing lives here too: the member checks every entity needs -and the walk over a configuration that applies them. What stays with the -entity is the table saying which members it has -- that is the part that -is about blosc rather than about entities. +The shared plumbing lives here too: the member checks every entity needs, +the compiler that reads them off a field annotation, and the walk over a +configuration that applies them. What stays with the entity is its fields +-- the part that is about blosc rather than about entities -- and +everything the layer knows about a member is read from those. """ from __future__ import annotations -from collections.abc import Mapping as _Mapping +# Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below +# (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class +# creation, and a name that exists only for the type checker is a NameError +# then -- for this package and for any tool introspecting an entity. +import types +from collections.abc import Callable, Mapping, Sequence from copy import deepcopy -from dataclasses import MISSING, Field, dataclass, fields +from dataclasses import MISSING, Field, dataclass, fields, is_dataclass from types import MappingProxyType from typing import ( TYPE_CHECKING, + Annotated, ClassVar, Final, Literal, NotRequired, - Protocol, Required, TypeAlias, TypeVar, + Union, cast, get_args, get_origin, get_type_hints, ) -from typing_extensions import ReadOnly, TypeIs +from typing_extensions import ReadOnly, TypeIs, is_typeddict from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET @@ -70,7 +77,6 @@ from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: - from collections.abc import Callable, Mapping, Sequence from typing import Self from zarr_metadata.model._validation import ProblemKind @@ -218,7 +224,7 @@ def _as_tuples(value: object) -> object: if isinstance(value, (list, tuple)): entries = cast("list[object] | tuple[object, ...]", value) return tuple(_as_tuples(entry) for entry in entries) - if isinstance(value, _Mapping): + if isinstance(value, Mapping): entries = cast("Mapping[str, object]", value) return {key: _as_tuples(entry) for key, entry in entries.items()} return value @@ -273,93 +279,344 @@ def coerce_members( return members, tuple(problems), frozenset(unreadable) -class ConfigurationType(Protocol): - """What this layer reads off a configuration TypedDict. +class _FromName: + """The marker behind `FROM_NAME`.""" - A Protocol rather than `type`, so that what is read off it is stated - rather than assumed. Deliberately not `__required_keys__`: under - `from __future__ import annotations` a TypedDict computes that from - unresolved strings and reports every member required, which would - make an extension's optional members silently mandatory. Requiredness - is read from the resolved annotation instead. + __slots__ = () + + def __repr__(self) -> str: + return "FROM_NAME" + + +FROM_NAME: Final = _FromName() +"""Marks a field carried by the metadata envelope's `name`, not its configuration. + + data_type_name: Annotated[str, FROM_NAME] + +A member all the same -- `value_problems` judges it -- but not a +configuration key, so it is neither read from nor written to a +`configuration` object. The raw-bytes family is the case: `r` keeps its +width in its name and has no configuration at all. +""" + + +def _strip(annotation: object) -> tuple[object, tuple[object, ...]]: + """An annotation's type, and the metadata `Annotated` wrapped it in. + + `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict + key, not part of the value's type; peeled with the `Annotated` layers, + in whatever order they were written. + """ + metadata: list[object] = [] + while True: + origin = get_origin(annotation) + if origin is Annotated: + inner, *extras = get_args(annotation) + metadata.extend(extras) + annotation = inner + elif origin in (NotRequired, Required, ReadOnly): + (annotation,) = get_args(annotation) + else: + return annotation, tuple(metadata) + + +def _field_hints(cls: type) -> dict[str, object]: + """The dataclass fields of `cls`, resolved, base first. + + Each class's own annotations are resolved in that class's module, + and class variables are skipped *before* resolving, by text -- so a + `ClassVar` whose annotation names something imported only for the + type checker cannot fail class creation. `@dataclass` sees the same + set, in the same order. + """ + hints: dict[str, object] = {} + for ancestor in reversed(cls.__mro__): + raw = { + name: annotation + for name, annotation in vars(ancestor).get("__annotations__", {}).items() + if not _is_class_var(annotation) + } + if len(raw) == 0: + continue + shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) + hints.update(get_type_hints(shell, include_extras=True)) + return hints + + +def _is_union(annotation: object) -> bool: + return get_origin(annotation) in (Union, types.UnionType) + + +def is_optional(annotation: object) -> bool: + """Whether a field may be absent: its type admits `UNSET`.""" + inner, _ = _strip(annotation) + return _is_union(inner) and any(arg is UNSET for arg in get_args(inner)) + + +def _is_entity_type(candidate: object) -> bool: + return candidate is Opaque or ( + isinstance(candidate, type) and issubclass(candidate, MetadataEntity) + ) + + +def _is_entity_or_opaque(candidates: Sequence[object]) -> bool: + """A nested metadata field: some entity kind, optionally with `Opaque`.""" + return ( + len(candidates) != 0 + and all(_is_entity_type(candidate) for candidate in candidates) + and any(candidate is not Opaque for candidate in candidates) + ) + + +def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """A nested metadata field: a bare name or a named-configuration object. + + Only the envelope's shape. Which entity the name denotes, and whether + its configuration is well formed, is settled when the containing + entity reads it in scope. """ + if not isinstance(value, (str, Mapping)): + return problem(loc, f"expected a metadata field, got {value!r}") + return () + - __name__: str +def describe(annotation: object) -> str: + """The annotation as a message would name it: "an integer", "an object".""" + inner, _ = _strip(annotation) + if inner is int: + return "an integer" + if inner is bool: + return "a boolean" + if inner is str: + return "a string" + if inner is JSONValue: + return "a JSON value" + origin = get_origin(inner) + if origin is Literal: + return f"one of {tuple(sorted(get_args(inner)))!r}" + if _is_union(inner): + branches = [arg for arg in get_args(inner) if arg is not UNSET] + if _is_entity_or_opaque(branches): + return "a metadata field" + return " or ".join(describe(branch) for branch in branches) + if origin is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + return f"an array of {describe(arguments[0])} elements" + if len(arguments) == 2: + return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" + return f"an array of {len(arguments)} elements" + if _is_entity_type(inner): + return "a metadata field" + if is_typeddict(inner) or is_dataclass(inner): + return "an object" + return "a value" -def _unwrap(annotation: object) -> object: - """An annotation without the qualifiers that are not its type. +def _shape(annotation: object) -> str | None: + """The top-level JSON shape an annotation admits, for choosing a union branch. - `NotRequired` and `Required` say whether a member must be present, - which is the other half of a member table entry; `ReadOnly` says - nothing about the value at all. + None means any shape -- a JSON value, or a union that mixes them. """ - while get_origin(annotation) in (NotRequired, Required, ReadOnly): - (annotation,) = get_args(annotation) - return annotation + inner, _ = _strip(annotation) + if inner is int: + return "int" + if inner is bool: + return "bool" + if inner is str: + return "str" + origin = get_origin(inner) + if origin is Literal: + values = get_args(inner) + return "int" if all(isinstance(value, int) for value in values) else "str" + if origin is tuple: + return "tuple" + if _is_entity_type(inner): + return "field" + if is_typeddict(inner) or is_dataclass(inner): + return "mapping" + return None -def check_for(annotation: object) -> TypeCheck | None: - """The check an annotation implies, or None if it implies none. +def _has_shape(shape: str | None, value: object) -> bool: + if shape is None: + return True + if shape == "int": + return is_integer(value) + if shape == "bool": + return isinstance(value, bool) + if shape == "str": + return isinstance(value, str) + if shape == "tuple": + return isinstance(value, tuple) + if shape == "mapping": + return isinstance(value, Mapping) + return isinstance(value, (str, Mapping)) # "field" + + +def any_of(branches: Sequence[tuple[object, TypeCheck]], description: str) -> TypeCheck: + """A member whose type is a union of shapes, judged by the branch it fits. + + The branch whose top-level shape the value has is the one that + reports -- so an element inside a malformed array is located inside + the array, rather than the whole array being called wrong. A value + fitting no branch's shape is reported once, by what was expected. + """ + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + fitting = [check for annotation, check in branches if _has_shape(_shape(annotation), value)] + if len(fitting) == 0: + return problem(loc, f"expected {description}, got {value!r}") + verdicts = [check(value, loc) for check in fitting] + return () if any(len(verdict) == 0 for verdict in verdicts) else verdicts[0] + + return check + + +def fixed_tuple(elements: Sequence[TypeCheck], description: str) -> TypeCheck: + """A member whose type is an array of a fixed length, checked position by position.""" - None for an annotation naming another structure -- a nested - TypedDict, a recursive JSON alias, a tuple of either. Reading those - off the annotation would be a TypedDict-to-checker compiler, which - is a different package; the entity declares those itself. + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple) or len(cast("tuple[object, ...]", value)) != len(elements): + return problem(loc, f"expected {description}, got {value!r}") + entries = cast("tuple[object, ...]", value) + return tuple( + found + for position, (element, entry) in enumerate(zip(elements, entries, strict=True)) + for found in element(entry, (*loc, position)) + ) + + return check + + +def mapping_of(members: Mapping[str, tuple[bool, TypeCheck]]) -> TypeCheck: + """A member that is itself an object with declared keys, checked key by key. + + Closed, like every configuration in this package: a key the type does + not declare is `unknown_key`, a required one missing is `missing_key`, + both located at the object. Each present member is checked at its own + key, so a problem inside is located inside. + """ + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, Mapping): + return problem(loc, f"expected an object, got {value!r}") + entries = cast("Mapping[str, object]", value) + found: list[ValidationProblem] = [] + for key in entries: + if key not in members: + found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) + for key, (required, member) in members.items(): + if key not in entries: + if required: + found.extend(problem(loc, f"missing required key {key!r}", "missing_key")) + continue + found.extend(member(entries[key], (*loc, key))) + return tuple(found) + + return check + + +def _members_of(annotations: Mapping[str, object]) -> dict[str, tuple[bool, TypeCheck]] | None: + """A member table for a nested object's keys; None if any key's type has no check.""" + members: dict[str, tuple[bool, TypeCheck]] = {} + for key, annotation in annotations.items(): + check = check_for(annotation) + if check is None: + return None + inner, _ = _strip(annotation) + required = get_origin(annotation) is not NotRequired and not is_optional(inner) + members[key] = (required, check) + return members + + +def check_for(annotation: object) -> TypeCheck | None: + """The type check a field annotation implies, or None if it implies none. + + A small compiler over the shapes this package's metadata takes: the + JSON scalars, a `Literal` of names, arrays homogeneous or fixed, + unions of those, a nested object described by a TypedDict or a + record dataclass, and a nested metadata field -- an entity type, + with or without `Opaque`. `UNSET` in a union says the member may be + absent, which is the other half of a table entry and is read + separately by `is_optional`. + + None for an annotation outside those shapes, which the entity then + declares a check for by hand. """ - annotation = _unwrap(annotation) - if annotation is int: + inner, _ = _strip(annotation) + if inner is int: return is_int - if annotation is bool: + if inner is bool: return is_bool - if annotation is str: + if inner is str: return is_str - if annotation is JSONValue: + if inner is JSONValue: return is_json_value - if get_origin(annotation) is Literal: + origin = get_origin(inner) + if origin is Literal: # Sorted, because the order `get_args` reports is not the order # the `Literal` was written in: two `Literal`s over the same # values compare and hash equal, so the first one built anywhere # in the process is the one every later one resolves to. The # check is a membership test either way; this is so the message # listing the values does not depend on import order. - return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(annotation))))) - if get_origin(annotation) is tuple: - arguments = get_args(annotation) + return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) + if _is_union(inner): + branches = [arg for arg in get_args(inner) if arg is not UNSET] + if len(branches) == 1: + return check_for(branches[0]) + if _is_entity_or_opaque(branches): + return is_metadata_field + compiled = [(branch, check_for(branch)) for branch in branches] + if any(check is None for _, check in compiled): + return None + return any_of( + [(branch, cast("TypeCheck", check)) for branch, check in compiled], describe(inner) + ) + if origin is tuple: + arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: element = check_for(arguments[0]) return None if element is None else sequence_of(element) + elements = [check_for(argument) for argument in arguments] + if any(element is None for element in elements): + return None + return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) + # A nested metadata field, before the record check: `Opaque` is itself + # a dataclass, and an entity type must not be walked as one either. + if _is_entity_type(inner): + return is_metadata_field + if is_typeddict(inner): + members = _members_of(get_type_hints(inner, include_extras=True)) + return None if members is None else mapping_of(members) + if isinstance(inner, type) and is_dataclass(inner): + members = _members_of(_field_hints(inner)) + return None if members is None else mapping_of(members) return None -def is_required(annotation: object) -> bool: - """Whether a configuration member must be present. - - From the resolved annotation rather than the TypedDict's - `__required_keys__`, which is computed from unresolved strings and - is wrong for a module using `from __future__ import annotations`. - `ReadOnly` may wrap either way round, so it is peeled first. - """ - while get_origin(annotation) is ReadOnly: - (annotation,) = get_args(annotation) - return get_origin(annotation) is not NotRequired - - -def derive_member_types( - configuration: ConfigurationType, -) -> dict[str, tuple[bool, TypeCheck]]: - """The member table a configuration TypedDict already describes. +def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], list[str]]: + """The member table an entity's own fields describe. - Requiredness is the TypedDict's, and so is the check wherever the - annotation implies one. A member it does not imply one for is left - out, for the entity to declare. + Every field is a configuration member unless `FROM_NAME` says it is + carried by the envelope. Requiredness is whether the type admits + `UNSET`; the check is whatever `check_for` reads off the type. Also + returned: the fields no check could be read for, which the entity + must declare by hand. """ derived: dict[str, tuple[bool, TypeCheck]] = {} - for member, annotation in get_type_hints(configuration, include_extras=True).items(): - check = check_for(annotation) - if check is not None: - derived[member] = (is_required(annotation), check) - return derived + unread: list[str] = [] + for name, annotation in _field_hints(cls).items(): + inner, metadata = _strip(annotation) + if any(entry is FROM_NAME for entry in metadata): + continue + check = check_for(inner) + if check is None: + unread.append(name) + continue + derived[name] = (not is_optional(inner), check) + return derived, unread ValueRoutine: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" @@ -479,23 +736,14 @@ class MetadataEntity: an invented identifier that no real name can collide with. """ - configuration_type: ClassVar[ConfigurationType | None] = None - """The TypedDict describing this entity's `configuration` in JSON. - - None for an entity that has no configuration. Everything else about - the members is read off it at class creation, so the JSON shape is - stated once: `member_types` and `configuration_required` are both - derived, and the constructor is held to the same keys by - `tests/v3/test_entities.py`. - """ - member_types: ClassVar[MemberTypes] = MappingProxyType({}) """The configuration members, and the type each one takes. - Derived from `configuration_type`. A class declares an entry here - only for a member whose annotation names another structure -- a - nested TypedDict, a recursive JSON alias -- which is where reading - the check off the annotation would take a compiler. + Read off the dataclass fields at class creation: which members there + are, which may be absent (the type admits `UNSET`), and the check + each one's type implies. A class declares an entry itself only for a + field whose annotation `check_for` cannot compile, and the public + JSON TypedDict is held to the same keys by `tests/v3/test_entities.py`. """ configuration_required: ClassVar[bool] = False @@ -503,7 +751,7 @@ class MetadataEntity: The spec permits a bare name "if no configuration metadata is required", so this is true exactly when some member is required -- - which the configuration TypedDict already says. + which the fields already say. """ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: @@ -543,40 +791,37 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: if "configuration_required" in vars(cls): msg = ( f"{cls.__name__} declares `configuration_required`, which follows " - "from whether its configuration has a required member" + "from whether any member is required" ) raise TypeError(msg) - if cls.configuration_type is not None: - # Before every guard below, because they read the table. - declared = dict(vars(cls).get("member_types", {})) - derived = derive_member_types(cls.configuration_type) - undeclared = sorted( - set(get_type_hints(cls.configuration_type)) - set(derived) - set(declared) + # Before every guard below, because they read the table. + declared = dict(vars(cls).get("member_types", {})) + derived, unread = derive_member_types(cls) + unsupported = sorted(set(unread) - set(declared)) + if len(unsupported) != 0: + msg = ( + f"{cls.__name__}: no check can be read off the annotation of " + f"{', '.join(unsupported)}; declare one in `member_types`" ) - if len(undeclared) != 0: - msg = ( - f"{cls.__name__} declares no check for {', '.join(undeclared)}, " - "whose annotation does not imply one" - ) - raise TypeError(msg) - hints = get_type_hints(cls.configuration_type, include_extras=True) - misstated = sorted( - member - for member, (required, _) in declared.items() - if member in hints and required != is_required(hints[member]) + raise TypeError(msg) + optional = {name: is_optional(annotation) for name, annotation in _field_hints(cls).items()} + misstated = sorted( + member + for member, (required, _) in declared.items() + if member in optional and required == optional[member] + ) + if len(misstated) != 0: + # The check is the entity's to write; whether the member may + # be left out is the field's to say, and a declared entry + # that disagrees is the drift this derivation exists to rule + # out. + msg = ( + f"{cls.__name__} declares {', '.join(misstated)} with a requiredness " + "its field does not give it" ) - if len(misstated) != 0: - # The check is the entity's to write; whether the member - # may be left out is the configuration's to say, and a - # declared entry that disagrees is the drift this - # derivation exists to rule out. - msg = ( - f"{cls.__name__} declares {', '.join(misstated)} with a requiredness " - "its configuration does not give it" - ) - raise TypeError(msg) - cls.member_types = {**derived, **declared} - cls.configuration_required = any(required for required, _ in cls.member_types.values()) + raise TypeError(msg) + cls.member_types = {**derived, **declared} + cls.configuration_required = any(required for required, _ in cls.member_types.values()) annotated = _declared_class_vars(cls) shadowed = [ name @@ -989,7 +1234,7 @@ def named_configuration( """ if isinstance(value, str): return value, None, True - if not isinstance(value, _Mapping): + if not isinstance(value, Mapping): return None, None, True entry = cast("Mapping[str, object]", value) name = entry.get("name") @@ -999,9 +1244,7 @@ def named_configuration( must_understand = entry.get("must_understand", True) return ( name, - cast("Mapping[str, object]", configuration) - if isinstance(configuration, _Mapping) - else None, + cast("Mapping[str, object]", configuration) if isinstance(configuration, Mapping) else None, must_understand if isinstance(must_understand, bool) else True, ) @@ -1011,6 +1254,7 @@ def named_configuration( "CHUNK_KEY_ENCODING", "CODECS", "DATA_TYPE", + "FROM_NAME", "STORAGE_TRANSFORMERS", "ChunkGridEntity", "CodecEntity", @@ -1030,6 +1274,7 @@ def named_configuration( "is_int", "is_integer", "is_json_value", + "is_metadata_field", "is_str", "named_configuration", "one_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 4eb945f0d8..8db7a6f6d2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -12,8 +12,6 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, - Loc, - MemberTypes, is_integer, problem, ) @@ -123,43 +121,6 @@ def canonical_chunk_shapes( ] -def _is_dim_specs(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """One spec per dimension, each a bare extent or a list of entries. - - An entry is an extent or a `[size, count]` run. The nesting is why - this is written out rather than composed from `sequence_of`. - """ - if not isinstance(value, tuple): - return problem(loc, f"expected an array of dimension specs, got {value!r}") - specs = cast("tuple[object, ...]", value) - found: list[ValidationProblem] = [] - for dim, spec in enumerate(specs): - at: Loc = (*loc, dim) - if is_integer(spec): - continue - if not isinstance(spec, tuple): - found.extend( - problem( - at, - "expected an integer or an array of integers / [value, count] pairs, " - f"got {spec!r}", - ) - ) - continue - for position, item in enumerate(cast("tuple[object, ...]", spec)): - if is_integer(item): - continue - entries = cast("tuple[object, ...]", item) if isinstance(item, tuple) else () - if len(entries) == 2 and all(is_integer(part) for part in entries): - continue - found.extend( - problem( - (*at, position), f"expected an integer or a [value, count] pair, got {item!r}" - ) - ) - return tuple(found) - - def _covered_extent(spec: tuple[int | tuple[int, int], ...]) -> int | None: """How much of a dimension an explicit spec covers, or None. @@ -208,11 +169,6 @@ class RectilinearChunkGrid(ChunkGridEntity): chunk_shapes: tuple[RectilinearDimSpec, ...] identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME - configuration_type = RectilinearChunkGridConfiguration - - member_types: ClassVar[MemberTypes] = { - "chunk_shapes": (True, _is_dim_specs), - } @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index f521339032..ad087d2d80 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -67,7 +67,6 @@ class RegularChunkGrid(ChunkGridEntity): chunk_shape: tuple[int, ...] identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME - configuration_type = RegularChunkGridConfiguration @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 59e6f3688b..a22b52383f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -77,7 +77,6 @@ class DefaultChunkKeyEncoding(MetadataEntity): separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME - configuration_type = DefaultChunkKeyEncodingConfiguration def to_json(self) -> DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName: return cast( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 337946a9d6..2abc10f872 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -83,7 +83,6 @@ class V2ChunkKeyEncoding(MetadataEntity): separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME - configuration_type = V2ChunkKeyEncodingConfiguration def to_json(self) -> V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName: return cast("V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 2ba444496a..4ab241b743 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -104,7 +104,6 @@ class BloscCodec(CodecEntity): typesize: int | UNSET = UNSET identifier: ClassVar[str] = BLOSC_CODEC_NAME - configuration_type = BloscCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 2552415ad6..8016c260fb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -90,7 +90,6 @@ class BytesCodec(CodecEntity): endian: Endianness | UNSET = UNSET identifier: ClassVar[str] = BYTES_CODEC_NAME - configuration_type = BytesCodecConfiguration kind: ClassVar[CodecKind] = "array_bytes" def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index f290ff53e9..1b9f869951 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -4,7 +4,6 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ -from collections.abc import Mapping from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast @@ -15,11 +14,7 @@ CodecEntity, CodecKind, DataTypeEntity, - Loc, - MemberTypes, Opaque, - is_json_value, - problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -136,43 +131,6 @@ class CastValueCodecObject(TypedDict, closed=True): """The two directions a `scalar_map` can override, both optional.""" -def _is_scalar_map(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """An object of `[old, new]` pairs per direction.""" - if not isinstance(value, Mapping): - return problem(loc, f"expected an object, got {value!r}") - mapping = cast("Mapping[str, object]", value) - found: list[ValidationProblem] = [] - for key in mapping: - if key not in SCALAR_MAP_KEYS: - found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) - for key in SCALAR_MAP_KEYS: - if key in mapping: - found.extend(_is_scalar_pairs(mapping[key], (*loc, key))) - return tuple(found) - - -def _is_scalar_pairs(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple): - return problem(loc, f"expected an array of [old, new] pairs, got {value!r}") - entries = cast("tuple[object, ...]", value) - found: list[ValidationProblem] = [] - for index, entry in enumerate(entries): - pair = cast("tuple[object, ...]", entry) if isinstance(entry, tuple) else () - if len(pair) != 2: - found.extend(problem((*loc, index), f"expected an [old, new] pair, got {entry!r}")) - continue - for position, scalar in enumerate(pair): - found.extend(is_json_value(scalar, (*loc, index, position))) - return tuple(found) - - -def _is_data_type_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """A metadata field -- which data type it names is settled on recursion.""" - if not isinstance(value, (str, Mapping)): - return problem(loc, f"expected a data type, got {value!r}") - return () - - @dataclass(frozen=True) class CastValueCodec(CodecEntity): """The `cast_value` codec, coerced from its metadata. @@ -187,14 +145,8 @@ class CastValueCodec(CodecEntity): scalar_map: ScalarMap | UNSET = UNSET identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME - configuration_type = CastValueCodecConfiguration kind: ClassVar[CodecKind] = "array_array" - member_types: ClassVar[MemberTypes] = { - "data_type": (True, _is_data_type_field), - "scalar_map": (False, _is_scalar_map), - } - @classmethod def prepare( cls, members: dict[str, object], context: "Context" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 7f45ce6af8..2dcfa95216 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -73,7 +73,6 @@ class GzipCodec(CodecEntity): level: int identifier: ClassVar[str] = GZIP_CODEC_NAME - configuration_type = GzipCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 4292c89ab1..8f2063d17a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -86,7 +86,6 @@ class ScaleOffsetCodec(CodecEntity): scale: JSONValue | UNSET = UNSET identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME - configuration_type = ScaleOffsetCodecConfiguration kind: ClassVar[CodecKind] = "array_array" def transition(self, incoming: ArrayParts) -> ArrayParts | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 898c4e487a..b8c789d0c4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -4,7 +4,6 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html """ -from collections.abc import Mapping from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast @@ -16,7 +15,6 @@ CodecEntity, CodecKind, Loc, - MemberTypes, Opaque, problem, ) @@ -102,19 +100,6 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): ] -def _is_field_tuple(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """An array of metadata fields -- their names are checked on recursion.""" - if not isinstance(value, tuple): - return problem(loc, f"expected an array of codecs, got {value!r}") - entries = cast("tuple[object, ...]", value) - return tuple( - found - for index, entry in enumerate(entries) - if not isinstance(entry, (str, Mapping)) - for found in problem((*loc, index), f"expected a metadata field, got {entry!r}") - ) - - def _coerce_pipeline( entries: tuple[object, ...], context: "Context", loc: Loc ) -> tuple[tuple[CodecEntity | Opaque, ...], tuple[ValidationProblem, ...]]: @@ -164,15 +149,9 @@ class ShardingIndexedCodec(CodecEntity): index_location: ShardingIndexLocation | UNSET = UNSET identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME - configuration_type = ShardingIndexedCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "array_bytes" - member_types: ClassVar[MemberTypes] = { - "codecs": (True, _is_field_tuple), - "index_codecs": (True, _is_field_tuple), - } - @staticmethod def value_problems( **members: Unpack[ShardingIndexedMembers], diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 670be8c237..c5164c5675 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -69,7 +69,6 @@ class TransposeCodec(CodecEntity): order: tuple[int, ...] identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME - configuration_type = TransposeCodecConfiguration kind: ClassVar[CodecKind] = "array_array" @staticmethod diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index a354289690..0fbf38fb38 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -82,7 +82,6 @@ class ZstdCodec(CodecEntity): checksum: bool | UNSET = UNSET identifier: ClassVar[str] = ZSTD_CODEC_NAME - configuration_type = ZstdCodecConfiguration variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index d2cd548f38..9256cd3aad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -82,7 +82,6 @@ class NumpyDatetime64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME - configuration_type = NumpyDatetime64Configuration @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index abf835c2bf..8b5d40c619 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -103,7 +103,6 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME - configuration_type = NumpyTimedelta64Configuration @staticmethod def value_problems( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index e9ddab0541..5816870596 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -10,13 +10,14 @@ import re from dataclasses import dataclass -from typing import ClassVar, Final, NewType, Self, cast +from typing import Annotated, ClassVar, Final, NewType, Self, cast from typing_extensions import TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( + FROM_NAME, Coerced, DataTypeEntity, Loc, @@ -119,7 +120,8 @@ class RawBytesDataType(DataTypeEntity): `r8`, and canonicalizing it away is not this package's call. """ - data_type_name: str + data_type_name: Annotated[str, FROM_NAME] + """The spelling as written -- `r8`, `r008` -- which is where the width lives.""" scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = RAW_BYTES_FAMILY diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 4c10e9ff44..c2575108d7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -13,7 +13,6 @@ DATA_TYPE, DataTypeEntity, Loc, - MemberTypes, Opaque, StorageClass, problem, @@ -89,34 +88,6 @@ class Struct(TypedDict, closed=True): ] -def _is_fields(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """An array of `{name, data_type}` objects. - - Whether a `data_type` names anything is settled on recursion; this - only asks whether the field entry has the two members at all. - """ - if not isinstance(value, tuple): - return problem(loc, f"expected an array of struct fields, got {value!r}") - entries = cast("tuple[object, ...]", value) - found: list[ValidationProblem] = [] - for index, entry in enumerate(entries): - if not isinstance(entry, Mapping): - found.extend(problem((*loc, index), f"expected a struct field, got {entry!r}")) - continue - field = cast("Mapping[str, object]", entry) - for key in STRUCT_FIELD_KEYS: - if key not in field: - found.extend(problem((*loc, index), f"missing required key {key!r}", "missing_key")) - for key in field: - if key not in STRUCT_FIELD_KEYS: - found.extend(problem((*loc, index), f"unexpected key {key!r}", "unknown_key")) - if "name" in field and not isinstance(field["name"], str): - found.extend( - problem((*loc, index, "name"), f"expected a string, got {field['name']!r}") - ) - return tuple(found) - - @dataclass(frozen=True) class StructFieldComponent: """One field of a struct: a name, and the type of its values. @@ -164,13 +135,8 @@ class StructDataType(DataTypeEntity): fields: tuple[StructFieldComponent, ...] identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME - configuration_type = StructConfiguration scalar_storage: ClassVar[StorageClass] = "single_byte" - member_types: ClassVar[MemberTypes] = { - "fields": (True, _is_fields), - } - @staticmethod def value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ...]: """What a struct can judge about its own fields. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 399e9fe97f..d0855b6bc4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -24,12 +24,9 @@ else: codec.json, codec.reason # 'out_of_scope': resolve it yourself -**Writing an extension.** Describe the JSON with a TypedDict, subclass -`CodecEntity`, `DataTypeEntity`, `ChunkGridEntity` or `MetadataEntity`, -point at the TypedDict, and add it to a scope: - - class AcmeLz4Configuration(TypedDict, closed=True): - acceleration: NotRequired[int] +**Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, +`ChunkGridEntity` or `MetadataEntity`, declare the fields, and add it to +a scope: @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): @@ -39,7 +36,6 @@ class AcmeLz4Codec(CodecEntity): identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_type = AcmeLz4Configuration SCOPE = CORE_AND_EXTENSIONS.extended_with( codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}, @@ -47,12 +43,17 @@ class AcmeLz4Codec(CodecEntity): validate_array_metadata_v3(document, context=SCOPE) -`configuration_type` is the only place the JSON shape is written. Which -members exist, which may be left out, and how each one is type-checked -are all read off it -- `member_types` is for the exception, a member -whose annotation names another structure. Value rules go in a -`value_problems` staticmethod annotated with the same TypedDict, which -runs only once every member has the type it declared: +The fields are the only place the shape is written. Which members exist, +which may be left out (the type admits `UNSET`), and how each one is +type-checked are all read off the annotations -- an `int`, a `Literal` +of names, an array, a nested entity type -- and `member_types` is for +the exception, an annotation the compiler does not read. Value rules go +in a `value_problems` staticmethod, which runs only once every member +has the type it declared; annotate it with a TypedDict of the members so +its body is checked: + + class AcmeLz4Configuration(TypedDict, closed=True): + acceleration: NotRequired[int] @staticmethod def value_problems( @@ -96,6 +97,7 @@ def value_problems( CHUNK_KEY_ENCODING, CODECS, DATA_TYPE, + FROM_NAME, STORAGE_TRANSFORMERS, ChunkGridEntity, CodecEntity, @@ -115,6 +117,7 @@ def value_problems( is_int, is_integer, is_json_value, + is_metadata_field, is_str, named_configuration, one_of, @@ -148,6 +151,7 @@ def value_problems( "CORE_AND_EXTENSIONS", "DATA_TYPE", "FLOAT_SPECIALS", + "FROM_NAME", "STORAGE_TRANSFORMERS", "UNKNOWN_GRID", "ArrayDocumentV3", @@ -183,6 +187,7 @@ def value_problems( "is_int", "is_integer", "is_json_value", + "is_metadata_field", "is_str", "named_configuration", "one_of", diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index a8b4d4fa3a..01c6164ec5 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -20,25 +20,29 @@ from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, + RectilinearChunkGridConfiguration, ) -from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid +from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid, RegularChunkGridConfiguration from zarr_metadata.v3.chunk_key_encoding.default import ( DefaultChunkKeyEncoding, + DefaultChunkKeyEncodingConfiguration, ) from zarr_metadata.v3.chunk_key_encoding.v2 import ( V2ChunkKeyEncoding, + V2ChunkKeyEncodingConfiguration, ) -from zarr_metadata.v3.codec.blosc import BloscCodec -from zarr_metadata.v3.codec.bytes import BytesCodec -from zarr_metadata.v3.codec.cast_value import CastValueCodec -from zarr_metadata.v3.codec.crc32c import Crc32cCodec -from zarr_metadata.v3.codec.gzip import GzipCodec -from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec +from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration +from zarr_metadata.v3.codec.bytes import BytesCodec, BytesCodecConfiguration +from zarr_metadata.v3.codec.cast_value import CastValueCodec, CastValueCodecConfiguration +from zarr_metadata.v3.codec.crc32c import Crc32cCodec, Empty +from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecConfiguration +from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec, ScaleOffsetCodecConfiguration from zarr_metadata.v3.codec.sharding_indexed import ( ShardingIndexedCodec, + ShardingIndexedCodecConfiguration, ) -from zarr_metadata.v3.codec.transpose import TransposeCodec -from zarr_metadata.v3.codec.zstd import ZstdCodec +from zarr_metadata.v3.codec.transpose import TransposeCodec, TransposeCodecConfiguration +from zarr_metadata.v3.codec.zstd import ZstdCodec, ZstdCodecConfiguration from zarr_metadata.v3.data_type.bool import BoolDataType from zarr_metadata.v3.data_type.bytes import BytesDataType from zarr_metadata.v3.data_type.complex64 import Complex64DataType @@ -51,29 +55,25 @@ from zarr_metadata.v3.data_type.int32 import Int32DataType from zarr_metadata.v3.data_type.int64 import Int64DataType from zarr_metadata.v3.data_type.numpy_datetime64 import ( + NumpyDatetime64Configuration, NumpyDatetime64DataType, ) from zarr_metadata.v3.data_type.numpy_timedelta64 import ( + NumpyTimedelta64Configuration, NumpyTimedelta64DataType, ) from zarr_metadata.v3.data_type.raw import RawBytesDataType from zarr_metadata.v3.data_type.string import StringDataType -from zarr_metadata.v3.data_type.struct import StructDataType +from zarr_metadata.v3.data_type.struct import StructConfiguration, StructDataType from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType from zarr_metadata.v3.entity import ArrayDocumentV3, MetadataEntity -# Each registered entity, paired with the TypedDict its constructor -# mirrors. Keyed by `:`, because an identifier is only -# unique within its extension point -- `bytes` is both a codec and a data -# type. # Every registered entity, keyed by `:` -- an identifier # is unique only within its extension point, and `bytes` is both a codec -# and a data type. What each one's configuration is comes off the class: -# `configuration_type` is the only place that says so, and the fields are -# held to it below. +# and a data type. ENTITIES: dict[str, type[MetadataEntity]] = { "codecs:blosc": BloscCodec, "codecs:bytes": BytesCodec, @@ -110,25 +110,48 @@ "data_type:r": RawBytesDataType, } +# The public JSON TypedDict each configured entity's fields must mirror. Test +# data, not a class attribute: nothing in the package reads it any more, so +# this is the one correspondence still written by hand -- and the one that +# catches an entity whose fields drift from the JSON type it is documented by. +# An entity absent here has no configuration. +CONFIGURATIONS: dict[str, type] = { + "codecs:blosc": BloscCodecConfiguration, + "codecs:bytes": BytesCodecConfiguration, + "codecs:cast_value": CastValueCodecConfiguration, + "codecs:crc32c": Empty, + "codecs:gzip": GzipCodecConfiguration, + "codecs:scale_offset": ScaleOffsetCodecConfiguration, + "codecs:sharding_indexed": ShardingIndexedCodecConfiguration, + "codecs:transpose": TransposeCodecConfiguration, + "codecs:zstd": ZstdCodecConfiguration, + "chunk_grid:regular": RegularChunkGridConfiguration, + "chunk_grid:rectilinear": RectilinearChunkGridConfiguration, + "chunk_key_encoding:default": DefaultChunkKeyEncodingConfiguration, + "chunk_key_encoding:v2": V2ChunkKeyEncodingConfiguration, + "data_type:numpy.datetime64": NumpyDatetime64Configuration, + "data_type:numpy.timedelta64": NumpyTimedelta64Configuration, + "data_type:struct": StructConfiguration, +} + @pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) def test_the_constructor_mirrors_the_configuration(entity: type[MetadataEntity]) -> None: - # The one correspondence still written by hand, and so the one that - # can still drift: the member table and `configuration_required` are - # now read off `configuration_type`, but the dataclass fields are - # not. It is also what catches an entity pointing at the wrong - # TypedDict, since the fields would stop matching. + # The member table and `configuration_required` are read off the fields, + # so the fields are the only spelling left that can drift from the public + # JSON TypedDict -- and a field the TypedDict does not have would be a + # member no document could write. # - # `must_understand` belongs to the object, not the configuration, so - # it is the one field the two deliberately do not share. - configuration = entity.configuration_type + # `must_understand` belongs to the object, not the configuration, so it + # is the one field the two deliberately do not share. + key = next(key for key, candidate in ENTITIES.items() if candidate is entity) fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} - if configuration is None: + if key not in CONFIGURATIONS: # `r` keeps its width in its name, so it holds a member that # is not a configuration key. assert fields == ({"data_type_name"} if entity is RawBytesDataType else set()) return - assert fields == set(get_type_hints(configuration)) + assert fields == set(get_type_hints(CONFIGURATIONS[key])) @pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 5a84a82571..a658360ff7 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -61,7 +61,6 @@ class AcmeLz4Codec(CodecEntity): identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" variable_size: ClassVar[bool] = True - configuration_type = AcmeLz4Configuration @staticmethod def value_problems( @@ -215,11 +214,12 @@ def test_error_an_optional_member_defaults_to_unset() -> None: @dataclass(frozen=True) class Inventive(CodecEntity): # pyright: ignore[reportUnusedClass] - level: int = 3 + # Optional by its type, so the annotation and the default agree + # on that much; it is the default's value that is wrong. + level: int | UNSET = 3 # pyright: ignore[reportAssignmentType] identifier: ClassVar[str] = "acme.inventive" kind: ClassVar[CodecKind] = "bytes_bytes" - member_types: ClassVar[MemberTypes] = {"level": (False, is_int)} def test_a_reader_gets_entities_or_an_exception() -> None: @@ -387,10 +387,10 @@ def test_a_third_party_can_register_a_family() -> None: def test_error_requiredness_may_not_be_restated() -> None: - # It is the configuration's to say. A declared entry exists for the - # check, which the annotation does not imply; saying the member is - # required as well is the drift the derivation removes. - with pytest.raises(TypeError, match="requiredness its configuration does not give it"): + # It is the field's to say. A declared entry exists for the check, + # which the annotation does not imply; saying the member is required + # as well is the drift the derivation removes. + with pytest.raises(TypeError, match="requiredness its field does not give it"): @dataclass(frozen=True) class Insistent(CodecEntity): # pyright: ignore[reportUnusedClass] @@ -398,18 +398,14 @@ class Insistent(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.insistent" kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_type = AcmeLz4Configuration member_types: ClassVar[MemberTypes] = {"acceleration": (True, is_int)} def test_error_a_member_needs_a_check_from_somewhere() -> None: - # An annotation naming another structure implies no check, so the - # entity owes one. Silently skipping the member would let anything - # through where the TypedDict promised a shape. - class Nested(TypedDict, closed=True): - inner: AcmeLz4Configuration - - with pytest.raises(TypeError, match="declares no check for inner"): + # An annotation outside the shapes `check_for` compiles implies no + # check, so the entity owes one. Silently skipping the member would + # let anything through where the field promised a type. + with pytest.raises(TypeError, match="no check can be read off the annotation of inner"): @dataclass(frozen=True) class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] @@ -417,12 +413,11 @@ class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.structured" kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_type = Nested def test_error_a_bare_name_rule_may_not_be_restated() -> None: # Whether the bare spelling is legal follows from whether any member - # is required, which the configuration already says. + # is required, which the fields already say. with pytest.raises(TypeError, match="declares `configuration_required`"): @dataclass(frozen=True) @@ -431,5 +426,4 @@ class Opinionated(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.opinionated" kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_type = AcmeLz4Configuration configuration_required: ClassVar[bool] = True From 6d0e00048e848d096365b14f533e37de17530d8e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 09:23:46 +0200 Subject: [PATCH 059/107] refactor(zarr-metadata): an entity that contains entities writes nothing for it `cast_value`, `sharding_indexed` and `struct` each wrote the same walk three times by hand: `prepare` to read a nested metadata field through the scope, `configuration` to write it back as JSON, `canonical` to recurse into it. The field annotation already said everything those needed -- `data_type: DataTypeEntity | Opaque`, `codecs: tuple[ CodecEntity | Opaque, ...]`, a record holding one -- so three walkers over the annotation replace the nine overrides, and the `prepare` hook goes with them. Each entity kind names its `extension_point`, which is what makes a nested field resolvable: the scope is asked at that point. A field typed as bare `MetadataEntity` is refused at class creation, since the two points that take any entity are no single point to ask. Defining `prepare` is refused the way `problems` and `__post_init__` are: it would be resolution that never runs. The two `canonical` overrides that remain -- blosc's ignored `typesize`, rectilinear's run-length encoding -- are about the entity's own members and start from `super().canonical()`, so the walk into contained entities is not lost. Over 40,000 documents: no problem, location or verdict differs. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 13 + .../src/zarr_metadata/v3/_entity.py | 292 ++++++++++++++++-- .../v3/chunk_grid/rectilinear.py | 2 +- .../src/zarr_metadata/v3/codec/blosc.py | 7 +- .../src/zarr_metadata/v3/codec/cast_value.py | 44 +-- .../v3/codec/sharding_indexed.py | 76 +---- .../src/zarr_metadata/v3/data_type/struct.py | 65 +--- .../tests/v3/test_extension_api.py | 101 ++++++ 8 files changed, 407 insertions(+), 193 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 880a24bf97..f250b96e37 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -53,3 +53,16 @@ fail class creation. One diagnostic became more precise: a malformed `[value, count]` pair in a rectilinear grid is now reported at the offending element inside the pair rather than at the pair. No verdict changes. + +An entity that contains other entities writes nothing for it. A field +annotated with an entity type -- `data_type: DataTypeEntity | Opaque`, +`codecs: tuple[CodecEntity | Opaque, ...]`, a record holding one -- is +resolved through the scope at the point that kind is registered at, +written back as each contained entity's own JSON, and put in canonical +form by recursing into it, all read off the annotation. The `prepare` +hook is gone, and so are the three `prepare`, three `configuration` and +three `canonical` overrides that did that walk by hand for `cast_value`, +`sharding_indexed` and `struct`. Each entity kind names its +`extension_point`, which is what makes a nested field resolvable; a +field typed as bare `MetadataEntity` is refused at class creation, since +no scope could place it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index d9a2f83f97..0720102474 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -46,7 +46,7 @@ import types from collections.abc import Callable, Mapping, Sequence from copy import deepcopy -from dataclasses import MISSING, Field, dataclass, fields, is_dataclass +from dataclasses import MISSING, Field, dataclass, fields, is_dataclass, replace from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -619,6 +619,191 @@ def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], l return derived, unread +def _contains_entity(annotation: object) -> bool: + """Whether a value of this type holds a nested metadata field anywhere in it.""" + inner, _ = _strip(annotation) + if _is_entity_type(inner): + return True + origin = get_origin(inner) + if _is_union(inner): + return any(_contains_entity(arg) for arg in get_args(inner) if arg is not UNSET) + if origin is tuple: + return any(_contains_entity(arg) for arg in get_args(inner) if arg is not Ellipsis) + if is_typeddict(inner): + return any( + _contains_entity(value) for value in get_type_hints(inner, include_extras=True).values() + ) + if isinstance(inner, type) and is_dataclass(inner): + return any(_contains_entity(value) for value in _field_hints(inner).values()) + return False + + +def _as_entity_kind(candidate: object) -> type[MetadataEntity] | None: + """`candidate` as an entity type, or None if it is not one. + + In a function of its own so that the `isinstance`/`issubclass` pair + narrows this parameter and not the caller's variable, which the + caller goes on to read as the annotation it is. + """ + if isinstance(candidate, type) and issubclass(candidate, MetadataEntity): + return candidate + return None + + +def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: + """Every entity type an annotation names, at any depth.""" + inner, _ = _strip(annotation) + kind = _as_entity_kind(inner) + if kind is not None: + return [kind] + origin = get_origin(inner) + arguments: tuple[object, ...] = get_args(inner) + if _is_union(inner): + return [kind for arg in arguments if arg is not UNSET for kind in _entity_kinds(arg)] + if origin is tuple: + return [kind for arg in arguments if arg is not Ellipsis for kind in _entity_kinds(arg)] + if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + return [kind for value in _field_hints(inner).values() for kind in _entity_kinds(value)] + return [] + + +def _point_of(kind: type[MetadataEntity]) -> ExtensionPointField: + """The point a nested entity kind is resolved at. + + Every nested field's kind has one by the time an entity exists -- + `__init_subclass__` refuses the class otherwise -- so this is the + narrowing, not a second check. + """ + point = kind.extension_point + if point is None: + msg = f"{kind.__name__} is registered at no single extension point" + raise TypeError(msg) + return point + + +def _element_annotations(inner: object, count: int) -> list[object]: + """The annotation of each element of a tuple type, one per element held.""" + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + return [arguments[0]] * count + return list(arguments) + + +def _fitting_branch(inner: object, value: object) -> object | None: + """The branch of a union that holds an entity and whose shape `value` has.""" + for branch in get_args(inner): + if branch is UNSET or not _contains_entity(branch): + continue + if _has_shape(_shape(branch), value): + return branch + return None + + +def _resolve( + annotation: object, value: object, context: Context, loc: Loc +) -> tuple[object, tuple[ValidationProblem, ...]]: + """`value`, with every nested metadata field in it read as an entity in `context`. + + What `prepare` used to be written for by hand: a field annotated with + an entity type is resolved through the scope, at the point that kind + of entity is registered at; an array of them element by element; a + record holding one field by field. The value has passed its type + check, so the shapes are the annotation's. + """ + inner, _ = _strip(annotation) + candidates = list(get_args(inner)) if _is_union(inner) else [inner] + if _is_entity_or_opaque(candidates): + return context.coerce(_point_of(_entity_kinds(inner)[0]), value, loc) + if _is_union(inner): + branch = _fitting_branch(inner, value) + return (value, ()) if branch is None else _resolve(branch, value, context, loc) + if get_origin(inner) is tuple: + entries = cast("tuple[object, ...]", value) + resolved: list[object] = [] + found: list[ValidationProblem] = [] + for position, (element, entry) in enumerate( + zip(_element_annotations(inner, len(entries)), entries, strict=True) + ): + item, problems = _resolve(element, entry, context, (*loc, position)) + resolved.append(item) + found.extend(problems) + return tuple(resolved), tuple(found) + if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + entries = cast("Mapping[str, object]", value) + members: dict[str, object] = {} + found = [] + for name, field_annotation in _field_hints(inner).items(): + if name not in entries: + continue + member, problems = _resolve(field_annotation, entries[name], context, (*loc, name)) + members[name] = member + found.extend(problems) + return inner(**members), tuple(found) + return value, () + + +def _render(annotation: object, value: object) -> object: + """`value` as a document would write it: every nested entity in its JSON form.""" + if isinstance(value, MetadataEntity): + return value.to_json() + if isinstance(value, Opaque): + return value.json + inner, _ = _strip(annotation) + if _is_union(inner): + branch = _fitting_branch(inner, value) + return value if branch is None else _render(branch, value) + if get_origin(inner) is tuple: + entries = cast("tuple[object, ...]", value) + return tuple( + _render(element, entry) + for element, entry in zip( + _element_annotations(inner, len(entries)), entries, strict=True + ) + ) + if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + return { + name: _render(field_annotation, getattr(value, name)) + for name, field_annotation in _field_hints(inner).items() + if getattr(value, name) is not UNSET + } + return value + + +def _canonicalize(annotation: object, value: object) -> object: + """`value` with every nested entity in its own canonical form.""" + if isinstance(value, MetadataEntity): + return value.canonical() + if isinstance(value, Opaque): + return value + inner, _ = _strip(annotation) + if _is_union(inner): + branch = _fitting_branch(inner, value) + return value if branch is None else _canonicalize(branch, value) + if get_origin(inner) is tuple: + entries = cast("tuple[object, ...]", value) + return tuple( + _canonicalize(element, entry) + for element, entry in zip( + _element_annotations(inner, len(entries)), entries, strict=True + ) + ) + if ( + isinstance(inner, type) + and is_dataclass(inner) + and not _is_entity_type(inner) + and is_dataclass(value) + and not isinstance(value, type) + ): + return replace( + value, + **{ + name: _canonicalize(field_annotation, getattr(value, name)) + for name, field_annotation in _field_hints(inner).items() + }, + ) + return value + + ValueRoutine: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" """An entity's value-space judgment, over the members it was given.""" @@ -712,6 +897,16 @@ class MetadataEntity: rather than a constant, a member another member renders meaningless. """ + extension_point: ClassVar[ExtensionPointField | None] = None + """Where this kind of entity is registered, if it is registered at one point. + + Set by `CodecEntity`, `DataTypeEntity` and `ChunkGridEntity`. It is + what makes a field typed as one of those resolvable: the scope is + asked at that point. `MetadataEntity` itself is the kind of the two + points that take any entity, so it names none, and a field typed as + bare `MetadataEntity` is refused at class creation. + """ + must_understand: ClassVar[bool] = True """Whether a reader that does not know this entity may skip it. @@ -746,6 +941,15 @@ class MetadataEntity: JSON TypedDict is held to the same keys by `tests/v3/test_entities.py`. """ + nested_members: ClassVar[Mapping[str, object]] = MappingProxyType({}) + """The fields that hold other entities, with their annotations. + + Read off the fields at class creation, like `member_types`. These are + the members `coerce` resolves through the scope, `configuration` + renders as JSON and `canonical` recurses into -- so an entity that + contains entities writes nothing for any of that. + """ + configuration_required: ClassVar[bool] = False """Whether the bare-name spelling says too little for this entity. @@ -777,6 +981,15 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: "`value_problems`, which takes the members rather than an entity" ) raise TypeError(msg) + if "prepare" in cls.__dict__: + # A member that is an entity is read from its annotation, so + # an override named `prepare` is resolution that would never + # run, and nothing else would say so. + msg = ( + f"{cls.__name__} defines `prepare`; a member that is an entity is read " + "from its field annotation, and nothing calls `prepare`" + ) + raise TypeError(msg) if "__post_init__" in cls.__dict__: # `coerce` builds through `unchecked`, which bypasses # `__init__` and so never reaches `__post_init__`. Rules put @@ -822,6 +1035,24 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: raise TypeError(msg) cls.member_types = {**derived, **declared} cls.configuration_required = any(required for required, _ in cls.member_types.values()) + hints = _field_hints(cls) + cls.nested_members = { + name: annotation for name, annotation in hints.items() if _contains_entity(annotation) + } + unplaced = sorted( + name + for name, annotation in cls.nested_members.items() + if any(kind.extension_point is None for kind in _entity_kinds(annotation)) + ) + if len(unplaced) != 0: + # `MetadataEntity` itself is registered at no single point, so + # a field typed as one could not be resolved through a scope. + msg = ( + f"{cls.__name__}: the entity kind of {', '.join(unplaced)} has no " + "`extension_point`; annotate it with `CodecEntity`, `DataTypeEntity` " + "or `ChunkGridEntity`" + ) + raise TypeError(msg) annotated = _declared_class_vars(cls) shadowed = [ name @@ -905,19 +1136,6 @@ def accepts(cls, name: str) -> bool: """ return name == cls.identifier - @classmethod - def prepare( - cls, members: dict[str, object], context: Context - ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: - """The members, with any that are themselves entities read as such. - - The seam between `coerce_members`, which knows types, and - `value_problems`, which knows values: a `struct` cannot ask - whether a field is fixed-size until that field's data type is an - entity. Default: nothing to convert. - """ - return members, () - @classmethod def coerce(cls, value: object, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one. @@ -940,8 +1158,12 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: if len(unreadable) == 0: # Before judging: a member that is itself an entity has to be # one before its container's value rules can ask it anything. - members, nested = cls.prepare(members, context) - found = (*found, *nested) + for name, annotation in cls.nested_members.items(): + if name in members: + members[name], nested = _resolve( + annotation, members[name], context, ("configuration", name) + ) + found = (*found, *nested) if len(unreadable) != 0: # A member that could not be read leaves a hole, and the value # rules are written over a whole configuration -- blosc's @@ -963,20 +1185,31 @@ def canonical(self) -> Self: a reader that reads and writes should not change bytes it was not asked to change. - Default: entities are already canonical. Override where two - spellings of a member mean the same -- a rectilinear dimension's - run-length encoding, a `typesize` that `noshuffle` ignores -- and - where a contained entity has its own canonical form. + A contained entity is put in its own canonical form here, by + walking the fields that hold one. Override where two spellings of + the entity's *own* members mean the same -- a rectilinear + dimension's run-length encoding, a `typesize` that `noshuffle` + ignores -- and start from `super().canonical()`, so the walk is + not lost. """ - return self + nested = type(self).nested_members + if len(nested) == 0: + return self + return replace( + self, + **{ + name: _canonicalize(annotation, getattr(self, name)) + for name, annotation in nested.items() + }, + ) def configuration(self) -> dict[str, object]: """This entity's configuration, as the document would write it. Faithful to every member the entity holds: `to_json` is serialization, not canonicalization, so nothing is simplified - here. Override only to render a member that is not already JSON, - such as a contained entity. + here. A contained entity is rendered as its own `to_json`, by + walking the fields that hold one. Absent optional members are left out, which is what makes the bare-name spelling reachable. Absence is `UNSET`, never `None`: @@ -989,7 +1222,11 @@ def configuration(self) -> dict[str, object]: the entity's own dict would let them mutate a frozen entity through the document it returned. """ - return deepcopy(self._configuration_members()) + members = self._configuration_members() + for name, annotation in type(self).nested_members.items(): + if name in members: + members[name] = _render(annotation, members[name]) + return deepcopy(members) value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) """Every value among the members the spec disallows. @@ -1119,6 +1356,9 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: class CodecEntity(MetadataEntity, base=True): """An entity that occupies a position in the codec pipeline.""" + extension_point: ClassVar[ExtensionPointField] = CODECS + """Where a codec is registered, and so where a field typed as one is resolved.""" + kind: ClassVar[CodecKind] variable_size: ClassVar[bool] = False @@ -1157,6 +1397,8 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: class ChunkGridEntity(MetadataEntity, base=True): """An entity that divides an array into the parts a pipeline encodes.""" + extension_point: ClassVar[ExtensionPointField] = CHUNK_GRID + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """Why this grid does not divide an array of `array_shape`. @@ -1185,6 +1427,8 @@ class DataTypeEntity(MetadataEntity, base=True): a table of names. """ + extension_point: ClassVar[ExtensionPointField] = DATA_TYPE + scalar_storage: ClassVar[StorageClass] def storage_class(self) -> StorageClass | None: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 8db7a6f6d2..a7c2f91087 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -257,7 +257,7 @@ def canonical(self) -> Self: Two dimension specs listing the same extents describe the same grid, and the encoded one stays the same size as the array grows. """ - return replace(self, chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) + return replace(super().canonical(), chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) def to_json(self) -> RectilinearChunkGridObject: return cast("RectilinearChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 4ab241b743..538199f4c0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -158,9 +158,10 @@ def canonical(self) -> Self: The spec says of that case that "the value is ignored", so two documents differing only there describe the same codec. """ - if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: - return self - return replace(self, typesize=UNSET) + canonical = super().canonical() + if canonical.shuffle != BLOSC_NO_SHUFFLE or canonical.typesize is UNSET: + return canonical + return replace(canonical, typesize=UNSET) def to_json(self) -> BloscCodecObject: return cast("BloscCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 1b9f869951..adbec3b57d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -4,13 +4,15 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ -from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast +from typing_extensions import TypedDict + +from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( - DATA_TYPE, CodecEntity, CodecKind, DataTypeEntity, @@ -18,14 +20,6 @@ ) from zarr_metadata.v3._parts import ArrayParts -if TYPE_CHECKING: - from zarr_metadata.v3._registry import Context - -from typing_extensions import TypedDict - -from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -147,32 +141,6 @@ class CastValueCodec(CodecEntity): identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME kind: ClassVar[CodecKind] = "array_array" - @classmethod - def prepare( - cls, members: dict[str, object], context: "Context" - ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: - """The target data type, read in this scope.""" - data_type, found = context.coerce( - DATA_TYPE, members["data_type"], ("configuration", "data_type") - ) - return {**members, "data_type": data_type}, found - - def canonical(self) -> Self: - """The target data type in its own canonical form.""" - if not isinstance(self.data_type, DataTypeEntity): - return self - return replace(self, data_type=self.data_type.canonical()) - - def configuration(self) -> dict[str, object]: - """The target data type in its canonical spelling.""" - members = super().configuration() - data_type = self.data_type - if isinstance(data_type, DataTypeEntity): - members["data_type"] = data_type.to_json() - else: - members["data_type"] = data_type.json - return members - def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index b8c789d0c4..0e34ce0d3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -4,17 +4,18 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html """ -from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast + +from typing_extensions import TypedDict, Unpack from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._chain import chain_problems +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( - CODECS, CodecEntity, CodecKind, - Loc, Opaque, problem, ) @@ -26,13 +27,6 @@ ) from zarr_metadata.v3.data_type.uint64 import Uint64DataType -if TYPE_CHECKING: - from zarr_metadata.v3._registry import Context - -from typing_extensions import TypedDict, Unpack - -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -100,32 +94,12 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): ] -def _coerce_pipeline( - entries: tuple[object, ...], context: "Context", loc: Loc -) -> tuple[tuple[CodecEntity | Opaque, ...], tuple[ValidationProblem, ...]]: - """Every entry of one pipeline, read in `context`.""" - coerced: list[CodecEntity | Opaque] = [] - problems: list[ValidationProblem] = [] - for index, entry in enumerate(entries): - codec, found = context.coerce(CODECS, entry, (*loc, index)) - coerced.append(codec) - problems.extend(found) - return tuple(coerced), tuple(problems) - - -def _canonical_pipeline( - codecs: tuple[CodecEntity | Opaque, ...], -) -> tuple[CodecEntity | Opaque, ...]: - """Each codec canonicalized; one out of scope is left as written.""" - return tuple(codec.canonical() if isinstance(codec, CodecEntity) else codec for codec in codecs) - - class ShardingIndexedMembers(TypedDict): """A shard's members as the entity holds them. Not `ShardingIndexedCodecConfiguration`, which describes the JSON: by - the time values are judged, `prepare` has read the two pipelines, so - these are codecs rather than the metadata fields that named them. + the time values are judged, the two pipelines have been read in scope, + so these are codecs rather than the metadata fields that named them. """ chunk_shape: tuple[int, ...] @@ -171,24 +145,6 @@ def value_problems( if extent < 1 ) - @classmethod - def prepare( - cls, members: dict[str, object], context: "Context" - ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: - """Both pipelines, read in this scope.""" - inner, from_inner = _coerce_pipeline( - cast("tuple[object, ...]", members["codecs"]), context, ("configuration", "codecs") - ) - index, from_index = _coerce_pipeline( - cast("tuple[object, ...]", members["index_codecs"]), - context, - ("configuration", "index_codecs"), - ) - return ( - {**members, "codecs": inner, "index_codecs": index}, - (*from_inner, *from_index), - ) - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """This shard against the array reaching it, and its two pipelines. @@ -261,23 +217,5 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation ) return tuple(found) - def canonical(self) -> Self: - """Each codec of each pipeline in its own canonical form.""" - return replace( - self, - codecs=_canonical_pipeline(self.codecs), - index_codecs=_canonical_pipeline(self.index_codecs), - ) - - def configuration(self) -> dict[str, object]: - """The two pipelines in their canonical spelling, entry by entry.""" - members = super().configuration() - for member in ("codecs", "index_codecs"): - members[member] = tuple( - entry.to_json() if isinstance(entry, CodecEntity) else entry.json - for entry in cast("tuple[CodecEntity | Opaque, ...]", members[member]) - ) - return members - def to_json(self) -> ShardingIndexedCodecObject: return cast("ShardingIndexedCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index c2575108d7..3add970dc8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -5,12 +5,15 @@ """ from collections.abc import Mapping -from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast +from dataclasses import dataclass +from typing import ClassVar, Final, Literal, NotRequired, cast +from typing_extensions import ReadOnly, TypedDict, Unpack + +from zarr_metadata._common import JSONValue from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( - DATA_TYPE, DataTypeEntity, Loc, Opaque, @@ -18,14 +21,6 @@ problem, ) -if TYPE_CHECKING: - from zarr_metadata.v3._registry import Context - -from typing_extensions import ReadOnly, TypedDict, Unpack - -from zarr_metadata._common import JSONValue -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -99,24 +94,12 @@ class StructFieldComponent: name: str data_type: DataTypeEntity | Opaque - def to_json(self) -> StructField: - data_type = self.data_type - return cast( - "StructField", - { - "name": self.name, - "data_type": ( - data_type.to_json() if isinstance(data_type, DataTypeEntity) else data_type.json - ), - }, - ) - class StructMembers(TypedDict): """A struct's members as the entity holds them. Not `StructConfiguration`, which describes the JSON: by the time - values are judged, `prepare` has read each field's data type, so + values are judged, each field's data type has been read in scope, so these are components holding entities rather than field objects. """ @@ -182,24 +165,6 @@ def value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ) return tuple(found) - @classmethod - def prepare( - cls, members: dict[str, object], context: "Context" - ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: - """Each field's data type, read in this scope.""" - fields: list[StructFieldComponent] = [] - found: list[ValidationProblem] = [] - for index, entry in enumerate(cast("tuple[object, ...]", members["fields"])): - field = cast("Mapping[str, object]", entry) - data_type, from_field = context.coerce( - DATA_TYPE, field["data_type"], ("configuration", "fields", index, "data_type") - ) - found.extend(from_field) - fields.append( - StructFieldComponent(name=cast("str", field["name"]), data_type=data_type) - ) - return {**members, "fields": tuple(fields)}, tuple(found) - def storage_class(self) -> StorageClass | None: """The widest class among the fields. @@ -255,21 +220,5 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP ) return tuple(found) - def canonical(self) -> Self: - """Each field's data type in its own canonical form.""" - return replace( - self, - fields=tuple( - replace(field, data_type=field.data_type.canonical()) - if isinstance(field.data_type, DataTypeEntity) - else field - for field in self.fields - ), - ) - - def configuration(self) -> dict[str, object]: - """Each field in its canonical spelling, type included.""" - return {"fields": tuple(field.to_json() for field in self.fields)} - def to_json(self) -> Struct: return cast("Struct", super().to_json()) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index a658360ff7..ee6c95a3d6 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -18,6 +18,8 @@ canonicalize_array_metadata_v3, validate_array_metadata_v3, ) +from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.entity import ( CORE, CORE_AND_EXTENSIONS, @@ -427,3 +429,102 @@ class Opinionated(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.opinionated" kind: ClassVar[CodecKind] = "bytes_bytes" configuration_required: ClassVar[bool] = True + + +# A third-party codec that contains another codec: the case that used to +# need `prepare`, `configuration` and `canonical` written by hand. +@dataclass(frozen=True) +class AcmeWrapperCodec(CodecEntity): + """A codec that applies another codec after its own step.""" + + inner: CodecEntity | Opaque + + identifier: ClassVar[str] = "acme.wrapper" + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> None: + # `inner: CodecEntity | Opaque` is the whole declaration. Reading it + # in scope, writing it back, and canonicalizing through it all follow + # from the annotation, so a wrapper is as short to write as a leaf. + scope = CORE_AND_EXTENSIONS.extended_with( + codecs={AcmeWrapperCodec.identifier: AcmeWrapperCodec} + ) + entry = { + "name": "acme.wrapper", + "configuration": {"inner": {"name": "gzip", "configuration": {"level": 5}}}, + } + codec, problems = scope.coerce("codecs", entry) + assert problems == () + assert isinstance(codec, AcmeWrapperCodec) + assert isinstance(codec.inner, GzipCodec) + assert codec.inner.level == 5 + assert codec.to_json() == entry + + # An inner codec the scope does not model stays verbatim, as anywhere. + unknown = {"name": "acme.wrapper", "configuration": {"inner": "acme.unknown"}} + codec, problems = scope.coerce("codecs", unknown) + assert problems == () + assert isinstance(codec, AcmeWrapperCodec) + assert isinstance(codec.inner, Opaque) + assert codec.to_json() == unknown + + # A problem inside is located inside. + bad = { + "name": "acme.wrapper", + "configuration": {"inner": {"name": "gzip", "configuration": {"level": 99}}}, + } + _, problems = scope.coerce("codecs", bad) + assert [problem.loc for problem in problems] == [ + ("configuration", "inner", "configuration", "level") + ] + + # Canonical form reaches the contained entity. + verbose = { + "name": "acme.wrapper", + "configuration": { + "inner": { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "noshuffle", + "typesize": 4, + "blocksize": 0, + }, + } + }, + } + codec, _ = scope.coerce("codecs", verbose) + assert isinstance(codec, AcmeWrapperCodec) + inner = codec.canonical().inner + assert isinstance(inner, BloscCodec) + assert inner.typesize is UNSET + + +def test_error_an_entity_may_not_define_prepare() -> None: + # A member that is an entity is read from its annotation; an override + # named `prepare` is resolution that would never run. + with pytest.raises(TypeError, match="nothing calls `prepare`"): + + @dataclass(frozen=True) + class Preparer(CodecEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.preparer" + kind: ClassVar[CodecKind] = "bytes_bytes" + + @classmethod + def prepare(cls, members: object, context: object) -> object: + return members + + +def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: + # `MetadataEntity` is registered at no single point, so a field typed + # as one could not be resolved through any scope. + with pytest.raises(TypeError, match="has no `extension_point`"): + + @dataclass(frozen=True) + class Vague(CodecEntity): # pyright: ignore[reportUnusedClass] + inner: MetadataEntity | Opaque + + identifier: ClassVar[str] = "acme.vague" + kind: ClassVar[CodecKind] = "bytes_bytes" From 872855a070450a173188173ca0bdb6a987b351d0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 09:31:26 +0200 Subject: [PATCH 060/107] refactor(zarr-metadata): a bound on a value is written on the field Seven value routines said nothing but a bound: gzip's and zstd's levels, the two time types' scale factors, and that every chunk extent -- in the regular grid, the sharding codec and the rectilinear grid, run-length counts included -- is at least one. Each was a routine unpacking the members, comparing one, and building a problem at a location it spelled by hand. The bound is on the field now, in the `annotated_types` vocabulary: `level: Annotated[int, Interval(ge=0, le=9)]`, `chunk_shape: tuple[Annotated[int, Ge(1)], ...]`. Defined here rather than imported, so the package keeps its one dependency; named as pydantic reads them and msgspec's `Meta` mirrors them, so a reader recognises them. The check is compiled at class creation and located at the member -- for an element, at its position -- and a bound on a union or record branch is applied through it, which is how rectilinear's `[value, count]` pairs carry the rule at both positions. One routine judges values on both paths: the annotation bounds, then `value_problems`, for `coerce` and the constructor alike. `value_problems` remains for a rule that is not a bound; five of those are left. Over 40,000 documents: no problem, location or verdict differs. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 12 + .../src/zarr_metadata/v3/_entity.py | 224 +++++++++++++++++- .../v3/chunk_grid/rectilinear.py | 56 +---- .../zarr_metadata/v3/chunk_grid/regular.py | 28 +-- .../src/zarr_metadata/v3/codec/gzip.py | 21 +- .../v3/codec/sharding_indexed.py | 40 +--- .../src/zarr_metadata/v3/codec/zstd.py | 23 +- .../v3/data_type/numpy_datetime64.py | 26 +- .../v3/data_type/numpy_timedelta64.py | 26 +- .../src/zarr_metadata/v3/entity.py | 22 +- .../tests/rules/test_chain_properties.py | 2 +- .../tests/rules/test_v3_array_rules.py | 6 +- .../zarr-metadata/tests/test_public_api.py | 5 + .../tests/v3/test_extension_api.py | 30 +-- 14 files changed, 301 insertions(+), 220 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index f250b96e37..1ed2ff2840 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -66,3 +66,15 @@ three `canonical` overrides that did that walk by hand for `cast_value`, `extension_point`, which is what makes a nested field resolvable; a field typed as bare `MetadataEntity` is refused at class creation, since no scope could place it. + +A bound on a value is written on the field, in the `annotated_types` +vocabulary -- `level: Annotated[int, Interval(ge=0, le=9)]`, +`chunk_shape: tuple[Annotated[int, Ge(1)], ...]` -- and judged at +whatever depth the annotation puts it, so a bound on an element type +locates its finding at the element. Seven value routines that stated +nothing but a bound are gone: gzip's and zstd's levels, the two time +types' scale factors, and the positivity of every chunk extent in the +regular grid, the sharding codec and the rectilinear grid, run-length +counts included. `value_problems` remains for a rule that is not a +bound. The bounds run with it, after the type checks, on the reading +path and in the constructor alike, through one routine. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 0720102474..90004082b6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -300,6 +300,52 @@ def __repr__(self) -> str: """ +@dataclass(frozen=True, slots=True) +class Ge: + """`Annotated[int, Ge(1)]`: the value is at least `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Gt: + """`Annotated[int, Gt(0)]`: the value is more than `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Le: + """`Annotated[int, Le(9)]`: the value is at most `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Lt: + """`Annotated[int, Lt(10)]`: the value is less than `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Interval: + """`Annotated[int, Interval(ge=0, le=9)]`: the value lies within these bounds. + + These five are the `annotated_types` vocabulary -- what pydantic reads + and msgspec's `Meta` mirrors -- so a reader recognises them. Defined + here rather than imported, so the package keeps its one dependency. + A bound is a value rule: it runs only once the member has the type it + declared, at whatever depth the annotation puts it, so a bound on an + array's element type judges each element at its own position. + """ + + ge: int | float | None = None + gt: int | float | None = None + le: int | float | None = None + lt: int | float | None = None + + def _strip(annotation: object) -> tuple[object, tuple[object, ...]]: """An annotation's type, and the metadata `Annotated` wrapped it in. @@ -619,6 +665,144 @@ def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], l return derived, unread +def _bound_check(metadata: Sequence[object]) -> TypeCheck | None: + """The check the bound markers among an annotation's metadata imply, or None.""" + ge = gt = le = lt = None + for marker in metadata: + if isinstance(marker, Ge): + ge = marker.bound + elif isinstance(marker, Gt): + gt = marker.bound + elif isinstance(marker, Le): + le = marker.bound + elif isinstance(marker, Lt): + lt = marker.bound + elif isinstance(marker, Interval): + ge = marker.ge if marker.ge is not None else ge + gt = marker.gt if marker.gt is not None else gt + le = marker.le if marker.le is not None else le + lt = marker.lt if marker.lt is not None else lt + if ge is None and gt is None and le is None and lt is None: + return None + if ge is not None and le is not None and gt is None and lt is None: + expectation = f"an integer in [{ge}, {le}]" + else: + comparisons = [ + text + for bound, text in ( + (ge, f">= {ge}"), + (gt, f"> {gt}"), + (le, f"<= {le}"), + (lt, f"< {lt}"), + ) + if bound is not None + ] + expectation = "an integer " + " and ".join(comparisons) + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + # Not a number: the type check's finding, not this one's. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return () + within_bounds = ( + (ge is None or value >= ge) + and (gt is None or value > gt) + and (le is None or value <= le) + and (lt is None or value < lt) + ) + if within_bounds: + return () + return problem(loc, f"expected {expectation}, got {value}", "invalid_value") + + return check + + +def value_check_for(annotation: object) -> TypeCheck | None: + """The value check an annotation's metadata implies, at any depth, or None. + + Over the shapes as the entity holds them, not as the JSON spells + them: this runs after every member has its type and every nested + entity has been read, so a record is a dataclass instance here and + an entity is skipped -- it is valid by construction. + """ + inner, metadata = _strip(annotation) + own = _bound_check(metadata) + below: TypeCheck | None = None + if _is_union(inner): + branches = [ + (branch, value_check_for(branch)) for branch in get_args(inner) if branch is not UNSET + ] + if any(check is not None for _, check in branches): + + def by_branch(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + for branch, check in branches: + if check is not None and _has_shape(_shape(branch), value): + return check(value, loc) + return () + + below = by_branch + elif get_origin(inner) is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = value_check_for(arguments[0]) + if element is not None: + each = element + + def per_element(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + entries = cast("tuple[object, ...]", value) + return tuple( + found + for position, entry in enumerate(entries) + for found in each(entry, (*loc, position)) + ) + + below = per_element + else: + positions = [value_check_for(argument) for argument in arguments] + if any(check is not None for check in positions): + + def per_position(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + entries = cast("tuple[object, ...]", value) + return tuple( + found + for position, (check, entry) in enumerate( + zip(positions, entries, strict=True) + ) + if check is not None + for found in check(entry, (*loc, position)) + ) + + below = per_position + elif isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + members = { + name: check + for name, field_annotation in _field_hints(inner).items() + if (check := value_check_for(field_annotation)) is not None + } + if len(members) != 0: + + def per_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + return tuple( + found + for name, check in members.items() + if (held := getattr(value, name)) is not UNSET + for found in check(held, (*loc, name)) + ) + + below = per_field + if own is None and below is None: + return None + if below is None: + return own + if own is None: + return below + outer, inner_check = own, below + + def both(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + return (*outer(value, loc), *inner_check(value, loc)) + + return both + + def _contains_entity(annotation: object) -> bool: """Whether a value of this type holds a nested metadata field anywhere in it.""" inner, _ = _strip(annotation) @@ -950,6 +1134,16 @@ class MetadataEntity: contains entities writes nothing for any of that. """ + value_checks: ClassVar[Mapping[str, TypeCheck]] = MappingProxyType({}) + """The value rules the field annotations state, member by member. + + A bound in an `Annotated` -- `level: Annotated[int, Interval(ge=0, + le=9)]`, `chunk_shape: tuple[Annotated[int, Ge(1)], ...]` -- becomes + a check here at class creation, located at the member and, for an + element, at its position. Runs with `value_problems`, after the type + checks, on both the reading path and the constructor. + """ + configuration_required: ClassVar[bool] = False """Whether the bare-name spelling says too little for this entity. @@ -1039,6 +1233,11 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: cls.nested_members = { name: annotation for name, annotation in hints.items() if _contains_entity(annotation) } + cls.value_checks = { + name: check + for name, annotation in hints.items() + if (check := value_check_for(annotation)) is not None + } unplaced = sorted( name for name, annotation in cls.nested_members.items() @@ -1170,7 +1369,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # `typesize` requirement reads `shuffle`. Judging around the # hole would be guessing, so the type problems stand alone. return None, found - found = (*found, *within((), cls.value_problems(**members))) + found = (*found, *within((), cls._judge_values(members))) if any(entry.kind != "unknown_key" for entry in found): return None, found # Already asked, so do not ask again on the way in. @@ -1254,10 +1453,26 @@ def __post_init__(self) -> None: than raising, and `unchecked` is the door for a caller that has already asked. """ - found = type(self).value_problems(**self._members()) + found = type(self)._judge_values(self._members()) if len(found) != 0: raise MetadataValidationError(found) + @classmethod + def _judge_values(cls, members: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + """Every value problem among the members. + + The bounds the annotations state first, member by member, then + whatever `value_problems` has to say -- one routine for the + reading path and the constructor, so the two cannot disagree. + """ + from_annotations = [ + found + for name, check in cls.value_checks.items() + if name in members + for found in check(members[name], (name,)) + ] + return (*from_annotations, *cls.value_problems(**members)) + def _members(self) -> dict[str, object]: """Every member this entity holds, unrendered. @@ -1506,7 +1721,12 @@ def named_configuration( "Coerced", "DataTypeEntity", "ExtensionPointField", + "Ge", + "Gt", + "Interval", + "Le", "Loc", + "Lt", "MemberTypes", "MetadataEntity", "Opaque", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index a7c2f91087..46115ece82 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -5,13 +5,14 @@ """ from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast +from typing import TYPE_CHECKING, Annotated, ClassVar, Final, Literal, NotRequired, Self, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, + Ge, is_integer, problem, ) @@ -42,6 +43,15 @@ pairs. """ +_PositiveExtent = Annotated[int, Ge(1)] +_PositiveDimSpec = ( + _PositiveExtent | tuple[_PositiveExtent | tuple[_PositiveExtent, _PositiveExtent], ...] +) +"""`RectilinearDimSpec` as the entity holds it: every extent, and every +run-length count, at least one. The same type to a type checker; the +bounds are what the reading path judges. +""" + class RectilinearChunkGridConfiguration(TypedDict, closed=True): """Configuration for the rectilinear chunk grid.""" @@ -166,50 +176,10 @@ class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" kind: Literal["inline"] - chunk_shapes: tuple[RectilinearDimSpec, ...] + chunk_shapes: tuple[_PositiveDimSpec, ...] identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME - @staticmethod - def value_problems( - **members: Unpack[RectilinearChunkGridConfiguration], - ) -> tuple[ValidationProblem, ...]: - """Every chunk extent, bare or run-length encoded, must be positive. - - A run's count must be positive too: a run of zero chunks is a way - of writing nothing at all, and the empty spelling already exists. - """ - found: list[ValidationProblem] = [] - for dim, spec in enumerate(members["chunk_shapes"]): - loc: tuple[str | int, ...] = ("chunk_shapes", dim) - if isinstance(spec, int): - if spec < 1: - found.extend( - problem( - loc, f"expected a positive chunk extent, got {spec}", "invalid_value" - ) - ) - continue - for position, item in enumerate(spec): - if isinstance(item, int): - if item < 1: - found.extend( - problem( - (*loc, position), - f"expected a positive chunk extent, got {item}", - "invalid_value", - ) - ) - elif item[0] < 1 or item[1] < 1: - found.extend( - problem( - (*loc, position), - f"expected a positive [size, count] pair, got {item!r}", - "invalid_value", - ) - ) - return tuple(found) - def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index ad087d2d80..63f08ade58 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -5,13 +5,14 @@ """ from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast +from typing import TYPE_CHECKING, Annotated, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, + Ge, problem, ) from zarr_metadata.v3._parts import ChunkGrid @@ -64,31 +65,10 @@ class RegularChunkGridObject(TypedDict, closed=True): class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" - chunk_shape: tuple[int, ...] + chunk_shape: tuple[Annotated[int, Ge(1)], ...] identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME - @staticmethod - def value_problems( - **members: Unpack[RegularChunkGridConfiguration], - ) -> tuple[ValidationProblem, ...]: - """Every chunk extent must be at least one element. - - A chunk of zero elements along an axis covers nothing, so no - finite number of them tiles the axis; a negative one is - meaningless. Whether there is one extent *per array dimension* is - a question for the document, and the rules layer asks it. - """ - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(members["chunk_shape"]) - if extent < 1 - ) - def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" if not isinstance(array_shape, (list, tuple)): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 2dcfa95216..cd90ff2372 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -5,15 +5,14 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict -from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - problem, + Interval, ) GZIP_CODEC_NAME: Final = "gzip" @@ -70,23 +69,11 @@ class GzipCodecObject(TypedDict, closed=True): class GzipCodec(CodecEntity): """The `gzip` codec, coerced from its metadata.""" - level: int + level: Annotated[int, Interval(ge=0, le=9)] identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" - @staticmethod - def value_problems( - **members: Unpack[GzipCodecConfiguration], - ) -> tuple[ValidationProblem, ...]: - """gzip compression levels run 0 to 9.""" - level = members["level"] - if not 0 <= level <= 9: - return problem( - ("level",), f"expected an integer in [0, 9], got {level}", "invalid_value" - ) - return () - def to_json(self) -> GzipCodecObject: return cast("GzipCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 0e34ce0d3f..8d18ffb38d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -5,9 +5,9 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem @@ -16,6 +16,7 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, + Ge, Opaque, problem, ) @@ -94,20 +95,6 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): ] -class ShardingIndexedMembers(TypedDict): - """A shard's members as the entity holds them. - - Not `ShardingIndexedCodecConfiguration`, which describes the JSON: by - the time values are judged, the two pipelines have been read in scope, - so these are codecs rather than the metadata fields that named them. - """ - - chunk_shape: tuple[int, ...] - codecs: tuple[CodecEntity | Opaque, ...] - index_codecs: tuple[CodecEntity | Opaque, ...] - index_location: NotRequired[ShardingIndexLocation] - - @dataclass(frozen=True) class ShardingIndexedCodec(CodecEntity): """The `sharding_indexed` codec, coerced from its metadata. @@ -117,7 +104,7 @@ class ShardingIndexedCodec(CodecEntity): itself an entity, read the same way this one was. """ - chunk_shape: tuple[int, ...] + chunk_shape: tuple[Annotated[int, Ge(1)], ...] codecs: tuple[CodecEntity | Opaque, ...] index_codecs: tuple[CodecEntity | Opaque, ...] index_location: ShardingIndexLocation | UNSET = UNSET @@ -126,25 +113,6 @@ class ShardingIndexedCodec(CodecEntity): variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "array_bytes" - @staticmethod - def value_problems( - **members: Unpack[ShardingIndexedMembers], - ) -> tuple[ValidationProblem, ...]: - """Every inner chunk extent must be at least one element. - - Nothing about the two pipelines: their codecs are entities, and an - entity exists only if its own values are allowed. - """ - return tuple( - ValidationProblem( - ("chunk_shape", position), - f"expected a positive chunk extent, got {extent}", - "invalid_value", - ) - for position, extent in enumerate(members["chunk_shape"]) - if extent < 1 - ) - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """This shard against the array reaching it, and its two pipelines. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 0fbf38fb38..34f9198243 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -7,16 +7,15 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - problem, + Interval, ) ZSTD_CODEC_NAME: Final = "zstd" @@ -78,26 +77,12 @@ class ZstdCodecObject(TypedDict, closed=True): class ZstdCodec(CodecEntity): """The `zstd` codec, coerced from its metadata.""" - level: int + level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] checksum: bool | UNSET = UNSET identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" - @staticmethod - def value_problems( - **members: Unpack[ZstdCodecConfiguration], - ) -> tuple[ValidationProblem, ...]: - """zstd compression levels run -131072 to 22.""" - level = members["level"] - if not ZSTD_MIN_LEVEL <= level <= ZSTD_MAX_LEVEL: - return problem( - ("level",), - f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {level}", - "invalid_value", - ) - return () - def to_json(self) -> ZstdCodecObject: return cast("ZstdCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 9256cd3aad..223f8c7f26 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -5,14 +5,13 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import ReadOnly, TypedDict, Unpack +from typing_extensions import ReadOnly, TypedDict -from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + Interval, StorageClass, - problem, ) from zarr_metadata.v3.data_type._families import NumpyTimeDataType from zarr_metadata.v3.data_type.numpy_timedelta64 import ( @@ -78,27 +77,10 @@ class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" unit: NumpyTimeUnit - scale_factor: int + scale_factor: Annotated[int, Interval(ge=1, le=NUMPY_TIME_MAX_SCALE_FACTOR)] scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME - @staticmethod - def value_problems( - **members: Unpack[NumpyDatetime64Configuration], - ) -> tuple[ValidationProblem, ...]: - """`scale_factor` counts units per step, so it is positive. - - The upper bound is numpy's: the field is a signed 32-bit integer. - """ - scale_factor = members["scale_factor"] - if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - return problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", - "invalid_value", - ) - return () - def to_json(self) -> NumpyDatetime64: return cast("NumpyDatetime64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 8b5d40c619..40698f9c75 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -5,14 +5,13 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import ReadOnly, TypedDict, Unpack +from typing_extensions import ReadOnly, TypedDict -from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + Interval, StorageClass, - problem, ) from zarr_metadata.v3.data_type._families import NumpyTimeDataType @@ -99,27 +98,10 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" unit: NumpyTimeUnit - scale_factor: int + scale_factor: Annotated[int, Interval(ge=1, le=NUMPY_TIME_MAX_SCALE_FACTOR)] scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME - @staticmethod - def value_problems( - **members: Unpack[NumpyTimedelta64Configuration], - ) -> tuple[ValidationProblem, ...]: - """`scale_factor` counts units per step, so it is positive. - - The upper bound is numpy's: the field is a signed 32-bit integer. - """ - scale_factor = members["scale_factor"] - if not 1 <= scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - return problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], got {scale_factor}", - "invalid_value", - ) - return () - def to_json(self) -> NumpyTimedelta64: return cast("NumpyTimedelta64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index d0855b6bc4..1715ca1600 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -47,10 +47,14 @@ class AcmeLz4Codec(CodecEntity): which may be left out (the type admits `UNSET`), and how each one is type-checked are all read off the annotations -- an `int`, a `Literal` of names, an array, a nested entity type -- and `member_types` is for -the exception, an annotation the compiler does not read. Value rules go -in a `value_problems` staticmethod, which runs only once every member -has the type it declared; annotate it with a TypedDict of the members so -its body is checked: +the exception, an annotation the compiler does not read. A bound on a +value is written on the field too, in the `annotated_types` vocabulary: + + acceleration: Annotated[int, Interval(ge=1, le=65537)] | UNSET = UNSET + +A rule that is not a bound goes in a `value_problems` staticmethod, +which runs only once every member has the type it declared; annotate it +with a TypedDict of the members so its body is checked: class AcmeLz4Configuration(TypedDict, closed=True): acceleration: NotRequired[int] @@ -105,7 +109,12 @@ def value_problems( Coerced, DataTypeEntity, ExtensionPointField, + Ge, + Gt, + Interval, + Le, Loc, + Lt, MemberTypes, MetadataEntity, Opaque, @@ -168,8 +177,13 @@ def value_problems( "ExtensionPointField", "Extents", "FloatDataType", + "Ge", + "Gt", "IntegerDataType", + "Interval", + "Le", "Loc", + "Lt", "MemberTypes", "MetadataEntity", "NumpyTimeDataType", diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py index 7ddf72309e..87fb998040 100644 --- a/packages/zarr-metadata/tests/rules/test_chain_properties.py +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -82,7 +82,7 @@ def test_a_well_ordered_chain_always_produces_a_verdict(codecs: tuple[object, .. # supplies. Those have their own tests in `test_v3_array_rules.py`. _WITNESSES: Mapping[str, str] = { "rank against the incoming array": "incoming array has", # 63% - "positive chunk extents": "expected a positive chunk extent", # 34% + "positive chunk extents": "expected an integer >= 1", # 34% "transpose order is a permutation": "expected a permutation", # 21% "endianness for multi-byte types": "endian is required", # 20% } diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index fcb7338225..54135ea129 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -135,7 +135,7 @@ def test_error_regular_chunk_extent_zero() -> None: {**BASE, "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (0, 2)}}} ) assert loc == ("chunk_grid", "configuration", "chunk_shape", 0) - assert "positive chunk extent" in message + assert "expected an integer >= 1" in message def test_error_rectilinear_rank_mismatch() -> None: @@ -176,7 +176,7 @@ def test_error_rectilinear_nonpositive_rle() -> None: }, } ) - assert any("positive [size, count] pair" in p.message for p in problems) + assert any("expected an integer >= 1" in p.message for p in problems) def test_error_transpose_not_a_permutation() -> None: @@ -272,7 +272,7 @@ def test_error_sharding_inner_chunk_extent_zero() -> None: problems = validate_array_metadata_v3({**BASE, "codecs": (_shard(chunk_shape=(0, 2)),)}) assert any( p.loc == ("codecs", 0, "configuration", "chunk_shape", 0) - and "positive chunk extent" in p.message + and "expected an integer >= 1" in p.message for p in problems ) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 913e18c69d..17d38fbfea 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -296,6 +296,11 @@ def test_all_is_grouped_and_unique() -> None: "MemberTypes", "Loc", "Extents", + "Ge", + "Gt", + "Interval", + "Le", + "Lt", "ExtensionPointField", "Context", "Coerced", diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index ee6c95a3d6..dd3b03850d 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,10 +8,9 @@ import re from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, NotRequired, Self, cast +from typing import TYPE_CHECKING, Annotated, ClassVar, Self, cast import pytest -from typing_extensions import TypedDict, Unpack from zarr_metadata.model import UNSET, MetadataValidationError, ValidationProblem from zarr_metadata.rules import ( @@ -32,6 +31,7 @@ Context, DataTypeEntity, IntegerDataType, + Interval, MemberTypes, MetadataEntity, Opaque, @@ -48,40 +48,16 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -class AcmeLz4Configuration(TypedDict, closed=True): - """The JSON shape of an `acme.lz4` configuration.""" - - acceleration: NotRequired[int] - - @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): """A third-party compressor.""" - acceleration: int | UNSET = UNSET + acceleration: Annotated[int, Interval(ge=1, le=ACME_MAX_ACCELERATION)] | UNSET = UNSET identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" variable_size: ClassVar[bool] = True - @staticmethod - def value_problems( - **members: Unpack[AcmeLz4Configuration], - ) -> tuple[ValidationProblem, ...]: - # No defensive narrowing: a member that failed its type check - # never reaches here, so asking whether it is present is enough - # to have an int. - if "acceleration" not in members: - return () - acceleration = members["acceleration"] - if not 1 <= acceleration <= ACME_MAX_ACCELERATION: - return problem( - ("acceleration",), - f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {acceleration}", - "invalid_value", - ) - return () - @dataclass(frozen=True) class AcmeFloat8DataType(DataTypeEntity): From 4791d9fce2b7d22d39b9c26a87eb3f4fa40e335d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 09:37:04 +0200 Subject: [PATCH 061/107] feat(zarr-metadata): a rule about one member is a @validates rule Four value routines remained after the bounds moved onto the fields. Three were about one member each -- transpose's `order` permuting itself, `scale_offset`'s `offset` and `scale` not being null, struct's `fields` forming a record -- and each unpacked `**members`, read the one it wanted, and spelled the location by hand. `@validates("order")` marks a staticmethod as the rule about that member. It receives the member's value, already of the declared type and only when present, and reports relative to the member; naming two members applies one rule to both, which is what `scale_offset` wanted. The decorator returns the function it was given and records it in a side table, so the declared signature survives and pyright keeps checking the body and any caller. Rules are collected at class creation, the nearest definition of a name winning, and a rule naming no field is refused. `value_problems` is now for a rule that reads two members together, which is exactly one: blosc's `typesize` against its `shuffle`. Its `clevel` and `blocksize` bounds moved onto the fields with the rest. Three places a value rule lives, by what it is about: a bound on the field, one member's rule under `@validates`, the members together in `value_problems`. One routine runs all three, on both paths. Over 40,000 documents: no problem, location or verdict differs. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 13 +++ .../src/zarr_metadata/v3/_entity.py | 96 ++++++++++++++++--- .../src/zarr_metadata/v3/codec/blosc.py | 36 +++---- .../zarr_metadata/v3/codec/scale_offset.py | 20 ++-- .../src/zarr_metadata/v3/codec/transpose.py | 13 +-- .../src/zarr_metadata/v3/data_type/struct.py | 44 ++++----- .../src/zarr_metadata/v3/entity.py | 19 +++- .../tests/rules/test_v3_array_rules.py | 2 +- .../tests/v3/test_extension_api.py | 59 +++++++++++- 9 files changed, 214 insertions(+), 88 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 1ed2ff2840..da62f59397 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -78,3 +78,16 @@ regular grid, the sharding codec and the rectilinear grid, run-length counts included. `value_problems` remains for a rule that is not a bound. The bounds run with it, after the type checks, on the reading path and in the constructor alike, through one routine. + +A rule about one member that is not a bound is a `@validates` rule: a +staticmethod that takes the member's value, runs only when the member +is present and has the type it declared, and reports relative to the +member. The decorator hands back the function it was given and records +it in a side table, so the declared signature survives for the type +checker. Transpose's permutation, `scale_offset`'s two not-null rules +(one rule, two members) and `struct`'s field rules are written that way. +`value_problems` remains for a rule that reads two members together, +which after this is exactly one: blosc's `typesize` against its +`shuffle`. Three places a value rule can live, by what it is about: a +bound on the field, a member's rule under `@validates`, the members +together in `value_problems`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 90004082b6..cafc941710 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -665,6 +665,42 @@ def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], l return derived, unread +MemberRule: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" +"""A rule about one member: takes its value, reports relative to it.""" + +_RULE_MEMBERS: Final[dict[object, tuple[str, ...]]] = {} +"""Which members each `@validates` rule is about, keyed by the function. + +A side table rather than an attribute on the function, so the decorator +hands back exactly what it was given -- the declared signature survives, +and the type checker keeps checking the body and its callers. +""" + +_Rule = TypeVar("_Rule", bound="Callable[..., tuple[ValidationProblem, ...]]") + + +def validates(*members: str) -> Callable[[_Rule], _Rule]: + """Mark a static rule as being about one member, or several alike. + + @staticmethod + @validates("order") + def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: + ... + + The rule receives the member's value, already of the type the field + declares, and only when the member is present; it reports relative + to the member, so a problem with an empty location is about the + member itself. Naming several members applies the one rule to each. + A rule that reads two members together is `value_problems`. + """ + + def mark(rule: _Rule) -> _Rule: + _RULE_MEMBERS[rule] = members + return rule + + return mark + + def _bound_check(metadata: Sequence[object]) -> TypeCheck | None: """The check the bound markers among an annotation's metadata imply, or None.""" ge = gt = le = lt = None @@ -1144,6 +1180,14 @@ class MetadataEntity: checks, on both the reading path and the constructor. """ + member_rules: ClassVar[Mapping[str, tuple[MemberRule, ...]]] = MappingProxyType({}) + """The `@validates` rules, by the member each is about. + + Collected at class creation from the class and its ancestors, the + nearest definition of a name winning. Run after the annotation bounds + and before `value_problems`, on both paths. + """ + configuration_required: ClassVar[bool] = False """Whether the bare-name spelling says too little for this entity. @@ -1171,8 +1215,9 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: # members. An override named `problems` is a rule that would # never run, and nothing else would say so. msg = ( - f"{cls.__name__} defines `problems`; value rules belong in " - "`value_problems`, which takes the members rather than an entity" + f"{cls.__name__} defines `problems`; a value rule is a bound on the " + "field, a `@validates` rule about one member, or `value_problems` " + "over the members together -- none of them takes an entity" ) raise TypeError(msg) if "prepare" in cls.__dict__: @@ -1238,6 +1283,22 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: for name, annotation in hints.items() if (check := value_check_for(annotation)) is not None } + attributes: dict[str, object] = {} + for ancestor in reversed(cls.__mro__): + attributes.update(vars(ancestor)) + rules: dict[str, list[MemberRule]] = {} + for attribute in attributes.values(): + function = attribute.__func__ if isinstance(attribute, staticmethod) else attribute + # Only a function can carry the mark; a table-valued class + # attribute is not even hashable. + if not callable(function): + continue + for member in _RULE_MEMBERS.get(function, ()): + if member not in hints: + msg = f"{cls.__name__}: `@validates({member!r})` names no field of the entity" + raise TypeError(msg) + rules.setdefault(member, []).append(cast("MemberRule", function)) + cls.member_rules = {member: tuple(found) for member, found in rules.items()} unplaced = sorted( name for name, annotation in cls.nested_members.items() @@ -1428,11 +1489,14 @@ def configuration(self) -> dict[str, object]: return deepcopy(members) value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) - """Every value among the members the spec disallows. - - A routine rather than a method, because judging values does not need - an entity -- and needing one would mean an invalid one had been - built. Each entity supplies its own, taking + """What the spec disallows among the members taken together. + + The third of three places a value rule lives, for the rule that + reads two members at once -- blosc's `typesize` against its + `shuffle`. A bound on one member is on the field; a rule about one + member is a `@validates` staticmethod. A routine rather than a + method, because judging values does not need an entity -- and + needing one would mean an invalid one had been built. Takes `Unpack[Configuration]`: the same spelling the constructor takes, receiving only the members that are present. @@ -1461,9 +1525,10 @@ def __post_init__(self) -> None: def _judge_values(cls, members: Mapping[str, object]) -> tuple[ValidationProblem, ...]: """Every value problem among the members. - The bounds the annotations state first, member by member, then - whatever `value_problems` has to say -- one routine for the - reading path and the constructor, so the two cannot disagree. + The bounds the annotations state first, then the `@validates` + rules, member by member, then whatever `value_problems` has to + say about the members together -- one routine for the reading + path and the constructor, so the two cannot disagree. """ from_annotations = [ found @@ -1471,7 +1536,14 @@ def _judge_values(cls, members: Mapping[str, object]) -> tuple[ValidationProblem if name in members for found in check(members[name], (name,)) ] - return (*from_annotations, *cls.value_problems(**members)) + from_rules = [ + ValidationProblem((member, *found.loc), found.message, found.kind) + for member, member_rules in cls.member_rules.items() + if member in members + for rule in member_rules + for found in rule(members[member]) + ] + return (*from_annotations, *from_rules, *cls.value_problems(**members)) def _members(self) -> dict[str, object]: """Every member this entity holds, unrendered. @@ -1727,6 +1799,7 @@ def named_configuration( "Le", "Loc", "Lt", + "MemberRule", "MemberTypes", "MetadataEntity", "Opaque", @@ -1744,5 +1817,6 @@ def named_configuration( "one_of", "problem", "sequence_of", + "validates", "within", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 538199f4c0..28cf6b28a9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass, replace -from typing import ClassVar, Final, Literal, NotRequired, Self, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import TypedDict, Unpack @@ -14,6 +14,8 @@ from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, + Ge, + Interval, problem, ) @@ -98,9 +100,9 @@ class BloscCodec(CodecEntity): """ cname: BloscCName - clevel: int + clevel: Annotated[int, Interval(ge=0, le=9)] shuffle: BloscShuffle - blocksize: int + blocksize: Annotated[int, Ge(0)] typesize: int | UNSET = UNSET identifier: ClassVar[str] = BLOSC_CODEC_NAME @@ -114,28 +116,16 @@ class BloscCodec(CodecEntity): def value_problems( **members: Unpack[BloscCodecConfiguration], ) -> tuple[ValidationProblem, ...]: - """The value constraints the spec places on a blosc configuration.""" - found: list[ValidationProblem] = [] - clevel = members["clevel"] - if not 0 <= clevel <= 9: - found.extend( - problem( - ("clevel",), f"expected an integer in [0, 9], got {clevel}", "invalid_value" - ) - ) - blocksize = members["blocksize"] - if blocksize < 0: - found.extend( - problem( - ("blocksize",), - f"expected a non-negative integer, got {blocksize}", - "invalid_value", - ) - ) + """`typesize` against `shuffle`: required, and positive, only where it counts. + + Under `noshuffle` the spec says of `typesize` that "the value is + ignored", and `canonical` drops it. A rule over two members, which + is what this routine is for; the bounds on `clevel` and + `blocksize` are on the fields. + """ shuffle = members["shuffle"] typesize = members.get("typesize") - # Only where it means something: under `noshuffle` the spec says - # "the value is ignored", and `canonical` drops it. + found: list[ValidationProblem] = [] if typesize is not None and shuffle != BLOSC_NO_SHUFFLE and typesize < 1: found.extend( problem( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 8f2063d17a..b7ee90cfad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET @@ -16,6 +16,7 @@ CodecEntity, CodecKind, problem, + validates, ) from zarr_metadata.v3._parts import ArrayParts @@ -97,9 +98,8 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming @staticmethod - def value_problems( - **members: Unpack[ScaleOffsetCodecConfiguration], - ) -> tuple[ValidationProblem, ...]: + @validates("offset", "scale") + def _is_a_scalar(value: JSONValue) -> tuple[ValidationProblem, ...]: """Each value is a scalar of the array's type, so neither is null. The registry says each is "JSON-encoded per the input array's @@ -107,15 +107,9 @@ def value_problems( Which scalar it should be needs the data type, so that part is the document's question, not this codec's. """ - # Each member named outright: a TypedDict indexed by a loop variable - # has no type, and the two are different members rather than two of - # a kind. - found: list[ValidationProblem] = [] - if members.get("offset", UNSET) is None: - found.extend(problem(("offset",), "expected a scalar, got null", "invalid_value")) - if members.get("scale", UNSET) is None: - found.extend(problem(("scale",), "expected a scalar, got null", "invalid_value")) - return tuple(found) + if value is None: + return problem((), "expected a scalar, got null", "invalid_value") + return () def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index c5164c5675..615e6791a2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -7,13 +7,14 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, problem, + validates, ) from zarr_metadata.v3._parts import ArrayParts @@ -72,20 +73,16 @@ class TransposeCodec(CodecEntity): kind: ClassVar[CodecKind] = "array_array" @staticmethod - def value_problems( - **members: Unpack[TransposeCodecConfiguration], - ) -> tuple[ValidationProblem, ...]: + @validates("order") + def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: """`order` must permute its own axes. Whether it permutes the *array's* axes is a different question -- it needs the array's rank -- and the rules layer asks that one. """ - order = members["order"] if sorted(order) != list(range(len(order))): return problem( - ("order",), - f"expected a permutation of 0..{len(order) - 1}, got {order!r}", - "invalid_value", + (), f"expected a permutation of 0..{len(order) - 1}, got {order!r}", "invalid_value" ) return () diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 3add970dc8..1db7adf145 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, cast -from typing_extensions import ReadOnly, TypedDict, Unpack +from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.model._validation import ValidationProblem @@ -19,6 +19,7 @@ Opaque, StorageClass, problem, + validates, ) STRUCT_DATA_TYPE_NAME: Final = "struct" @@ -95,17 +96,6 @@ class StructFieldComponent: data_type: DataTypeEntity | Opaque -class StructMembers(TypedDict): - """A struct's members as the entity holds them. - - Not `StructConfiguration`, which describes the JSON: by the time - values are judged, each field's data type has been read in scope, so - these are components holding entities rather than field objects. - """ - - fields: tuple[StructFieldComponent, ...] - - @dataclass(frozen=True) class StructDataType(DataTypeEntity): """The `struct` data type, coerced from its metadata. @@ -121,33 +111,31 @@ class StructDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "single_byte" @staticmethod - def value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ...]: - """What a struct can judge about its own fields. - - Names have to exist, be non-empty and be distinct, because a fill - value addresses fields by name. Field types have to be fixed-size, - because a record's layout is otherwise not determined. Nothing about - a field type's own values: it is an entity, so it exists only if - those are allowed. + @validates("fields") + def _fields_form_a_record( + fields: tuple[StructFieldComponent, ...], + ) -> tuple[ValidationProblem, ...]: + """Names exist, are non-empty and distinct; types are fixed-size. + + A fill value addresses fields by name, and a record's layout is + not determined by a variable-length field. Nothing about a field + type's own values: it is an entity, so it exists only if those + are allowed. """ - fields = members["fields"] found: list[ValidationProblem] = [] if len(fields) == 0: - found.extend( - problem(("fields",), "expected at least one struct field", "invalid_value") - ) + found.extend(problem((), "expected at least one struct field", "invalid_value")) seen: dict[str, int] = {} for index, field in enumerate(fields): - at: Loc = ("fields", index) if field.name == "": found.extend( - problem((*at, "name"), "expected a non-empty field name", "invalid_value") + problem((index, "name"), "expected a non-empty field name", "invalid_value") ) first = seen.setdefault(field.name, index) if first != index: found.extend( problem( - (*at, "name"), + (index, "name"), f"duplicate field name {field.name!r}, already used by field {first}", "invalid_value", ) @@ -158,7 +146,7 @@ def value_problems(**members: Unpack[StructMembers]) -> tuple[ValidationProblem, ): found.extend( problem( - (*at, "data_type"), + (index, "data_type"), "struct fields must use fixed-size data types", "invalid_value", ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 1715ca1600..380dead5da 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -52,9 +52,20 @@ class AcmeLz4Codec(CodecEntity): acceleration: Annotated[int, Interval(ge=1, le=65537)] | UNSET = UNSET -A rule that is not a bound goes in a `value_problems` staticmethod, -which runs only once every member has the type it declared; annotate it -with a TypedDict of the members so its body is checked: +A rule about one member that is not a bound is a `@validates` rule: a +staticmethod taking the member's value, run only when the member is +present and has the type it declared, reporting relative to the member: + + @staticmethod + @validates("order") + def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: + if sorted(order) != list(range(len(order))): + return problem((), f"expected a permutation, got {order!r}", "invalid_value") + return () + +A rule that reads two members together goes in a `value_problems` +staticmethod; annotate it with a TypedDict of the members so its body is +checked: class AcmeLz4Configuration(TypedDict, closed=True): acceleration: NotRequired[int] @@ -132,6 +143,7 @@ def value_problems( one_of, problem, sequence_of, + validates, within, ) from zarr_metadata.v3._parts import UNKNOWN_GRID, ArrayParts, ChunkGrid, Extents, shard_index_grid @@ -210,5 +222,6 @@ def value_problems( "read_array_v3", "sequence_of", "shard_index_grid", + "validates", "within", ] diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 54135ea129..4145ce8280 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -726,7 +726,7 @@ def test_error_blosc_clevel_out_of_range() -> None: def test_error_blosc_blocksize_is_negative() -> None: loc, message = _sole_problem(_with_blosc(blocksize=-1)) assert loc == ("codecs", 1, "configuration", "blocksize") - assert "non-negative" in message + assert "expected an integer >= 0" in message @pytest.mark.parametrize( diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index dd3b03850d..e964fe2c46 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -39,6 +39,7 @@ is_int, named_configuration, problem, + validates, ) ACME_MAX_ACCELERATION = 65537 @@ -264,7 +265,7 @@ def test_a_reader_can_choose_its_own_scope() -> None: def test_error_value_rules_must_be_value_problems() -> None: # `problems` was the old name and takes an entity; an override using # it would never run, and nothing else would notice. - with pytest.raises(TypeError, match="value rules belong in `value_problems`"): + with pytest.raises(TypeError, match="none of them takes an entity"): @dataclass(frozen=True) class Stale(CodecEntity): # pyright: ignore[reportUnusedClass] @@ -504,3 +505,59 @@ class Vague(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.vague" kind: ClassVar[CodecKind] = "bytes_bytes" + + +# A third-party rule about one member, written as a `@validates` rule. +@dataclass(frozen=True) +class AcmeBlockCodec(CodecEntity): + """A codec whose block size must be a power of two.""" + + block: int + + identifier: ClassVar[str] = "acme.block" + kind: ClassVar[CodecKind] = "bytes_bytes" + + @staticmethod + @validates("block") + def _block_is_a_power_of_two(block: int) -> tuple[ValidationProblem, ...]: + if block < 1 or block & (block - 1) != 0: + return problem((), f"expected a power of two, got {block}", "invalid_value") + return () + + +def test_a_rule_about_one_member_is_a_validates_rule() -> None: + # The rule receives the typed member, only when present, and reports + # relative to it: the location is supplied, and no `**members` is + # unpacked by hand. The declared signature survives, so a call by + # name is checked. + scope = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeBlockCodec.identifier: AcmeBlockCodec}) + codec, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": 64}}) + assert problems == () + assert isinstance(codec, AcmeBlockCodec) + _, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": 6}}) + assert [(p.loc, p.message) for p in problems] == [ + (("configuration", "block"), "expected a power of two, got 6") + ] + # And on the constructor, the same rule. + with pytest.raises(MetadataValidationError) as caught: + AcmeBlockCodec(block=6) + assert [p.loc for p in caught.value.problems] == [("block",)] + # A member that failed its type check never reaches the rule. + _, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": "x"}}) + assert [p.kind for p in problems] == ["invalid_type"] + + +def test_error_a_validates_rule_must_name_a_field() -> None: + with pytest.raises(TypeError, match="`@validates\\('blocc'\\)` names no field"): + + @dataclass(frozen=True) + class Misspelt(CodecEntity): # pyright: ignore[reportUnusedClass] + block: int + + identifier: ClassVar[str] = "acme.misspelt" + kind: ClassVar[CodecKind] = "bytes_bytes" + + @staticmethod + @validates("blocc") + def _rule(block: int) -> tuple[ValidationProblem, ...]: + return () From 3c08c407b824852bbd1445831c9542de3085b91c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 09:40:10 +0200 Subject: [PATCH 062/107] feat(zarr-metadata): the shapes the compiler reads are open `check_for` was a chain of `if`s over the annotation shapes this package happens to use. A third party with a field type outside them -- a `NewType`, a class of their own -- had one recourse: declare a check in `member_types` on every entity that uses it. Each shape is a registration now: a predicate over the annotation and what to compile it to, consulted in order. The ten built-in shapes are registered through the same door, in the order they must be tried, and `register_check` puts a registration from outside ahead of them, so the newest wins and a built-in shape can be re-judged too. cattrs' `register_structure_hook_func` is the pattern. Over 40,000 documents: no problem, location or verdict differs. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 9 + .../src/zarr_metadata/v3/_entity.py | 177 +++++++++++++----- .../src/zarr_metadata/v3/entity.py | 4 + .../zarr-metadata/tests/test_public_api.py | 1 + .../tests/v3/test_extension_api.py | 63 ++++++- 5 files changed, 203 insertions(+), 51 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index da62f59397..a1bae27f8b 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -91,3 +91,12 @@ which after this is exactly one: blosc's `typesize` against its `shuffle`. Three places a value rule can live, by what it is about: a bound on the field, a member's rule under `@validates`, the members together in `value_problems`. + +The set of annotation shapes the compiler reads is open. Each built-in +shape is a registration -- a predicate over the annotation and what to +compile it to -- consulted in order, and `register_check` adds one from +outside, ahead of the built-ins, so a package with a field type this +package does not read (a `NewType`, say) teaches the compiler once +rather than declaring a check on every entity that uses it. The same +door the built-in shapes came through; cattrs' `register_structure_hook_func` +is the pattern. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index cafc941710..57dc011380 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -576,6 +576,124 @@ def _members_of(annotations: Mapping[str, object]) -> dict[str, tuple[bool, Type return members +CheckCompiler: TypeAlias = "Callable[[object], TypeCheck | None]" +"""Turns one annotation into its type check -- or None, to decline it after all.""" + +_CHECK_COMPILERS: Final[list[tuple[Callable[[object], bool], CheckCompiler]]] = [] +"""The shapes `check_for` reads, each as (does this annotation have it?, compile it). + +Consulted front to back. The built-in shapes are appended below in the +order they must be tried -- a nested metadata field before a record, +because `Opaque` is itself a dataclass -- and `register_check` puts a +registration in front of all of them, so the newest one wins. +""" + + +def register_check(predicate: Callable[[object], bool], compile: CheckCompiler) -> None: + """Teach `check_for` an annotation shape it does not read. + + Hex = NewType("Hex", str) + register_check(lambda annotation: annotation is Hex, lambda annotation: is_hex) + + `predicate` sees the annotation with `Annotated`, `NotRequired` and + `ReadOnly` peeled; `compile` returns the check for it, calling + `check_for` itself for any shape inside. A registration is consulted + before every built-in one, so a package can also replace how a + built-in shape is judged. The same door the built-ins came through, + which is what makes the set of shapes open rather than this module's. + """ + _CHECK_COMPILERS.insert(0, (predicate, compile)) + + +def _builtin(predicate: Callable[[object], bool]) -> Callable[[CheckCompiler], CheckCompiler]: + """Register a built-in shape, in the order written.""" + + def append(compile: CheckCompiler) -> CheckCompiler: + _CHECK_COMPILERS.append((predicate, compile)) + return compile + + return append + + +@_builtin(lambda inner: inner is int) +def _compile_int(inner: object) -> TypeCheck | None: + return is_int + + +@_builtin(lambda inner: inner is bool) +def _compile_bool(inner: object) -> TypeCheck | None: + return is_bool + + +@_builtin(lambda inner: inner is str) +def _compile_str(inner: object) -> TypeCheck | None: + return is_str + + +@_builtin(lambda inner: inner is JSONValue) +def _compile_json_value(inner: object) -> TypeCheck | None: + return is_json_value + + +@_builtin(lambda inner: get_origin(inner) is Literal) +def _compile_literal(inner: object) -> TypeCheck | None: + # Sorted, because the order `get_args` reports is not the order the + # `Literal` was written in: two `Literal`s over the same values + # compare and hash equal, so the first one built anywhere in the + # process is the one every later one resolves to. The check is a + # membership test either way; this is so the message listing the + # values does not depend on import order. + return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) + + +@_builtin(_is_union) +def _compile_union(inner: object) -> TypeCheck | None: + branches = [arg for arg in get_args(inner) if arg is not UNSET] + if len(branches) == 1: + return check_for(branches[0]) + if _is_entity_or_opaque(branches): + return is_metadata_field + compiled = [(branch, check_for(branch)) for branch in branches] + if any(check is None for _, check in compiled): + return None + return any_of( + [(branch, cast("TypeCheck", check)) for branch, check in compiled], describe(inner) + ) + + +@_builtin(lambda inner: get_origin(inner) is tuple) +def _compile_tuple(inner: object) -> TypeCheck | None: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = check_for(arguments[0]) + return None if element is None else sequence_of(element) + elements = [check_for(argument) for argument in arguments] + if any(element is None for element in elements): + return None + return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) + + +# A nested metadata field, before the record shape: `Opaque` is itself a +# dataclass, and an entity type must not be walked as one either. +@_builtin(_is_entity_type) +def _compile_entity(inner: object) -> TypeCheck | None: + return is_metadata_field + + +@_builtin(is_typeddict) +def _compile_typeddict(inner: object) -> TypeCheck | None: + members = _members_of(get_type_hints(inner, include_extras=True)) + return None if members is None else mapping_of(members) + + +@_builtin(lambda inner: isinstance(inner, type) and is_dataclass(inner)) +def _compile_record(inner: object) -> TypeCheck | None: + if not isinstance(inner, type): # pragma: no cover - the predicate says it is + return None + members = _members_of(_field_hints(inner)) + return None if members is None else mapping_of(members) + + def check_for(annotation: object) -> TypeCheck | None: """The type check a field annotation implies, or None if it implies none. @@ -587,58 +705,15 @@ def check_for(annotation: object) -> TypeCheck | None: absent, which is the other half of a table entry and is read separately by `is_optional`. - None for an annotation outside those shapes, which the entity then - declares a check for by hand. + Open: each shape is a registration in `_CHECK_COMPILERS`, and + `register_check` adds one from outside. None for an annotation no + registration claims, which the entity then declares a check for by + hand. """ inner, _ = _strip(annotation) - if inner is int: - return is_int - if inner is bool: - return is_bool - if inner is str: - return is_str - if inner is JSONValue: - return is_json_value - origin = get_origin(inner) - if origin is Literal: - # Sorted, because the order `get_args` reports is not the order - # the `Literal` was written in: two `Literal`s over the same - # values compare and hash equal, so the first one built anywhere - # in the process is the one every later one resolves to. The - # check is a membership test either way; this is so the message - # listing the values does not depend on import order. - return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) - if _is_union(inner): - branches = [arg for arg in get_args(inner) if arg is not UNSET] - if len(branches) == 1: - return check_for(branches[0]) - if _is_entity_or_opaque(branches): - return is_metadata_field - compiled = [(branch, check_for(branch)) for branch in branches] - if any(check is None for _, check in compiled): - return None - return any_of( - [(branch, cast("TypeCheck", check)) for branch, check in compiled], describe(inner) - ) - if origin is tuple: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - element = check_for(arguments[0]) - return None if element is None else sequence_of(element) - elements = [check_for(argument) for argument in arguments] - if any(element is None for element in elements): - return None - return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) - # A nested metadata field, before the record check: `Opaque` is itself - # a dataclass, and an entity type must not be walked as one either. - if _is_entity_type(inner): - return is_metadata_field - if is_typeddict(inner): - members = _members_of(get_type_hints(inner, include_extras=True)) - return None if members is None else mapping_of(members) - if isinstance(inner, type) and is_dataclass(inner): - members = _members_of(_field_hints(inner)) - return None if members is None else mapping_of(members) + for predicate, compile in _CHECK_COMPILERS: + if predicate(inner): + return compile(inner) return None @@ -1787,6 +1862,7 @@ def named_configuration( "DATA_TYPE", "FROM_NAME", "STORAGE_TRANSFORMERS", + "CheckCompiler", "ChunkGridEntity", "CodecEntity", "CodecKind", @@ -1816,6 +1892,7 @@ def named_configuration( "named_configuration", "one_of", "problem", + "register_check", "sequence_of", "validates", "within", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 380dead5da..49d810ff31 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -114,6 +114,7 @@ def value_problems( DATA_TYPE, FROM_NAME, STORAGE_TRANSFORMERS, + CheckCompiler, ChunkGridEntity, CodecEntity, CodecKind, @@ -142,6 +143,7 @@ def value_problems( named_configuration, one_of, problem, + register_check, sequence_of, validates, within, @@ -177,6 +179,7 @@ def value_problems( "UNKNOWN_GRID", "ArrayDocumentV3", "ArrayParts", + "CheckCompiler", "ChunkGrid", "ChunkGridEntity", "CodecEntity", @@ -220,6 +223,7 @@ def value_problems( "order_problems", "problem", "read_array_v3", + "register_check", "sequence_of", "shard_index_grid", "validates", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 17d38fbfea..9967c72590 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -288,6 +288,7 @@ def test_all_is_grouped_and_unique() -> None: "BloscCName", "BloscShuffle", "Canonical", + "CheckCompiler", "CastOutOfRangeMode", "CastRoundingMode", "CodecKind", diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index e964fe2c46..f4b481bb26 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,7 +8,7 @@ import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, ClassVar, Self, cast +from typing import TYPE_CHECKING, Annotated, ClassVar, NewType, Self, cast import pytest @@ -17,6 +17,7 @@ canonicalize_array_metadata_v3, validate_array_metadata_v3, ) +from zarr_metadata.v3._entity import _CHECK_COMPILERS from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.entity import ( @@ -32,6 +33,7 @@ DataTypeEntity, IntegerDataType, Interval, + Loc, MemberTypes, MetadataEntity, Opaque, @@ -39,6 +41,7 @@ is_int, named_configuration, problem, + register_check, validates, ) @@ -561,3 +564,61 @@ class Misspelt(CodecEntity): # pyright: ignore[reportUnusedClass] @validates("blocc") def _rule(block: int) -> tuple[ValidationProblem, ...]: return () + + +# An annotation shape the compiler does not read, taught to it from outside. +Hex = NewType("Hex", str) + + +def _is_hex(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, str) or any(c not in "0123456789abcdef" for c in value): + return problem(loc, f"expected lowercase hex digits, got {value!r}") + return () + + +def test_a_third_party_can_teach_the_compiler_a_shape() -> None: + # `NewType` is a real case: to the type checker `Hex` is a `str`, but + # at run time it is a function `check_for` has no registration for, + # so an entity using it is refused -- until one is registered, through + # the same door the built-in shapes came through. + with pytest.raises(TypeError, match="no check can be read off the annotation of digest"): + + @dataclass(frozen=True) + class Unregistered(CodecEntity): # pyright: ignore[reportUnusedClass] + digest: Hex + + identifier: ClassVar[str] = "acme.unregistered" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def is_hex_annotation(annotation: object) -> bool: + return annotation is Hex + + register_check(is_hex_annotation, lambda annotation: _is_hex) + try: + + @dataclass(frozen=True) + class AcmeDigestCodec(CodecEntity): + digest: Hex + + identifier: ClassVar[str] = "acme.digest" + kind: ClassVar[CodecKind] = "bytes_bytes" + + scope = CORE_AND_EXTENSIONS.extended_with( + codecs={AcmeDigestCodec.identifier: AcmeDigestCodec} + ) + codec, problems = scope.coerce( + "codecs", {"name": "acme.digest", "configuration": {"digest": "c0ffee"}} + ) + assert problems == () + assert isinstance(codec, AcmeDigestCodec) + _, problems = scope.coerce( + "codecs", {"name": "acme.digest", "configuration": {"digest": "C0FFEE"}} + ) + assert [(p.loc, p.kind) for p in problems] == [ + (("configuration", "digest"), "invalid_type") + ] + finally: + # A registration is process-wide; leave the compiler as it was found. + _CHECK_COMPILERS[:] = [ + entry for entry in _CHECK_COMPILERS if entry[0] is not is_hex_annotation + ] From 955b01eaf73d54b818533066b71e5ba17c2761d3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 11:15:18 +0200 Subject: [PATCH 063/107] fix(zarr-metadata): read a class's annotations the way 3.14 hands them out From 3.14 a class does not carry an `__annotations__` dict until asked for one (PEP 649), so `vars(cls).get("__annotations__", {})` found no fields on any entity and class creation refused the first `@validates` rule it met. The 3.14 job caught it; every local run was on 3.11. `_own_annotations` asks `annotationlib` for the class's own annotations as the text they were written as, which is what the callers want anyway: class variables are skipped by text before anything is evaluated, so a `ClassVar` naming something imported only for the type checker still cannot fail the class. Earlier versions read the dict the class body left, as before. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_entity.py | 30 +++++-- .../tests/v3/test_extension_points.py | 82 ------------------- .../tests/v3/test_shape_properties.py | 57 ------------- 3 files changed, 24 insertions(+), 145 deletions(-) delete mode 100644 packages/zarr-metadata/tests/v3/test_extension_points.py delete mode 100644 packages/zarr-metadata/tests/v3/test_shape_properties.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 57dc011380..960a54ee74 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -43,6 +43,7 @@ # (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class # creation, and a name that exists only for the type checker is a NameError # then -- for this package and for any tool introspecting an entity. +import sys import types from collections.abc import Callable, Mapping, Sequence from copy import deepcopy @@ -366,6 +367,25 @@ def _strip(annotation: object) -> tuple[object, tuple[object, ...]]: return annotation, tuple(metadata) +def _own_annotations(klass: type) -> dict[str, object]: + """A class's own annotations, unevaluated. + + From 3.14 a class does not carry an `__annotations__` dict until it is + asked for one, and asking evaluates every annotation at once -- so a + `ClassVar` naming something imported only for the type checker would + fail the whole class. `annotationlib` can hand them back as the text + they were written as, which is what the callers here want anyway: + class variables are skipped by text before anything is evaluated. + Earlier versions leave the dict on the class, strings or values as + the module chose. + """ + if sys.version_info >= (3, 14): + import annotationlib + + return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) + return dict(vars(klass).get("__annotations__", {})) + + def _field_hints(cls: type) -> dict[str, object]: """The dataclass fields of `cls`, resolved, base first. @@ -379,7 +399,7 @@ def _field_hints(cls: type) -> dict[str, object]: for ancestor in reversed(cls.__mro__): raw = { name: annotation - for name, annotation in vars(ancestor).get("__annotations__", {}).items() + for name, annotation in _own_annotations(ancestor).items() if not _is_class_var(annotation) } if len(raw) == 0: @@ -1155,7 +1175,7 @@ def _declared_class_vars(cls: type) -> dict[str, type]: """ found: dict[str, type] = {} for ancestor in reversed(cls.__mro__): - for name, annotation in vars(ancestor).get("__annotations__", {}).items(): + for name, annotation in _own_annotations(ancestor).items(): if _is_class_var(annotation): found[name] = ancestor return found @@ -1391,10 +1411,8 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: annotated = _declared_class_vars(cls) shadowed = [ name - for name in vars(cls).get("__annotations__", {}) - if name in annotated - and annotated[name] is not cls - and not _is_class_var(vars(cls)["__annotations__"][name]) + for name, annotation in _own_annotations(cls).items() + if name in annotated and annotated[name] is not cls and not _is_class_var(annotation) ] if len(shadowed) != 0: # A field of that name would go into `member_types`, into the diff --git a/packages/zarr-metadata/tests/v3/test_extension_points.py b/packages/zarr-metadata/tests/v3/test_extension_points.py deleted file mode 100644 index b3355f205a..0000000000 --- a/packages/zarr-metadata/tests/v3/test_extension_points.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Tests for how a name reaches the entity that answers for it.""" - -from __future__ import annotations - -import pytest - -from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.v3._extension_points import ( - CODECS, - DATA_TYPE, - RAW_BYTES_FAMILY, -) -from zarr_metadata.v3.codec.blosc import BloscCodec -from zarr_metadata.v3.data_type.numpy_datetime64 import NumpyDatetime64DataType -from zarr_metadata.v3.data_type.raw import RawBytesDataType -from zarr_metadata.v3.data_type.uint8 import Uint8DataType -from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS, MetadataEntity - -# (field, name, the entity that answers for it — None when nothing does) -RESOLUTIONS: dict[str, tuple[str, str, type[MetadataEntity] | None]] = { - "plain-dtype": (DATA_TYPE, "uint8", Uint8DataType), - "dotted-dtype": (DATA_TYPE, "numpy.datetime64", NumpyDatetime64DataType), - "raw-8": (DATA_TYPE, "r8", RawBytesDataType), - "raw-24": (DATA_TYPE, "r24", RawBytesDataType), - # A malformed member reaches the family too: a misspelling of - # something we model must be reported as such, not pass as an unknown - # third-party extension. - "raw-not-multiple-of-8": (DATA_TYPE, "r12", RawBytesDataType), - "raw-zero": (DATA_TYPE, "r0", RawBytesDataType), - # Tables are per point, so the family cannot be reached from another. - "raw-shaped-codec-name": (CODECS, "r8", None), - "codec": (CODECS, "blosc", BloscCodec), - "unknown": (CODECS, "zfpy", None), - # The family's key is invented, so no document may write it. - "the-family-key-itself": (DATA_TYPE, RAW_BYTES_FAMILY, None), -} - - -@pytest.mark.parametrize(("field", "name", "expected"), RESOLUTIONS.values(), ids=list(RESOLUTIONS)) -def test_a_name_resolves_to_the_entity_that_answers_for_it( - field: str, name: str, expected: type[MetadataEntity] | None -) -> None: - assert CORE_AND_EXTENSIONS.resolve(field, name) is expected # type: ignore[arg-type] - - -def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: - # Zarr identifiers are registry-allocated. A private codec named - # `bytes` has left the compatibility contract, and saying so is the - # correct answer rather than a limitation, so nothing here defends - # against collisions. - document = { - "zarr_format": 3, - "node_type": "array", - "shape": (4, 4), - "data_type": "uint8", - "fill_value": 0, - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, - "chunk_key_encoding": "default", - "codecs": ({"name": "bytes", "configuration": {"width": 7}},), - } - problems = validate_array_metadata_v3(document) - assert [(p.loc, p.kind) for p in problems] == [ - (("codecs", 0, "configuration", "width"), "unknown_key") - ] - - -def test_forging_the_family_sentinel_cannot_change_a_verdict() -> None: - # A literal "r" data type mislabels nothing: the family claims its - # names through `accepts`, not through the table key, so no validation - # verdict depends on the sentinel being unforgeable. - document = { - "zarr_format": 3, - "node_type": "array", - "shape": (4, 4), - "data_type": RAW_BYTES_FAMILY, - "fill_value": (1,), - "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, - "chunk_key_encoding": "default", - "codecs": ("bytes",), - } - # Unjudged as an unknown data type, exactly as any unmodelled name is. - assert validate_array_metadata_v3(document) == () diff --git a/packages/zarr-metadata/tests/v3/test_shape_properties.py b/packages/zarr-metadata/tests/v3/test_shape_properties.py deleted file mode 100644 index 6140d92d72..0000000000 --- a/packages/zarr-metadata/tests/v3/test_shape_properties.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Generative invariants for how a family is resolved. - -Scoped to resolution deliberately. Two earlier tests here asserted that a -shape verdict exists exactly when `(field, canonical_name(...))` is in -`modelled_entities()` — but both sides were computed from `_ENTITY_SHAPES` -through the same call, so they restated the lookup rather than testing it, -and could not fail. Worse, they could not catch the bug class they named -(a lookup passing the wrong field), because both sides used the same -field. `tests/rules/test_registry.py` covers that with real assertions. - -A family is a genuine fit for generative testing: it is unbounded, so an -example-based test can only sample it. -""" - -from __future__ import annotations - -from hypothesis import given -from hypothesis import strategies as st - -from zarr_metadata.v3.data_type.raw import RawBytesDataType -from zarr_metadata.v3.entity import ( - CHUNK_GRID, - CODECS, - CORE_AND_EXTENSIONS, - DATA_TYPE, -) - - -@given(width=st.integers(min_value=0, max_value=2**32)) -def test_every_numeric_r_spelling_resolves_to_the_family(width: int) -> None: - # Including malformed widths (0, 12, anything not a multiple of 8): - # the family claims a name by grammar shape, not by validity, so a - # misspelled member of a family we model is reported as a misspelling - # rather than passing as an unknown third-party extension. - assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, f"r{width}") is RawBytesDataType - - -@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from([CODECS, CHUNK_GRID])) -def test_r_shaped_names_resolve_to_nothing_outside_data_types(width: int, field: str) -> None: - # The family belongs to `data_type`; a codec that happens to be named - # `r8` must not reach it. - assert CORE_AND_EXTENSIONS.resolve(field, f"r{width}") is None # type: ignore[arg-type] - - -# The scan `resolve` falls back to asks every entity, so a name no entity -# claims has to come back as nothing however many are registered. -_UNCLAIMED = st.text(min_size=1).filter( - lambda name: ( - not (name.startswith("r") and name[1:].isdigit()) - and name not in CORE_AND_EXTENSIONS.entities["data_type"] - ) -) - - -@given(name=_UNCLAIMED) -def test_a_name_no_entity_claims_resolves_to_nothing(name: str) -> None: - assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, name) is None From 4ba53d4a9e37b8344261d8cbd3ec3e1d17e53ae7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 11:16:37 +0200 Subject: [PATCH 064/107] refactor(zarr-metadata): the modules that had stopped earning their place `v3/_extension_points.py` existed to fold `r` names onto the family's key; since an entity claims its own names it had become a re-export of six names from `_entity`, so it goes and `_registry` imports them from where they live. `numpy_datetime64` and `numpy_timedelta64` each defined the time-unit vocabulary and imported the scale-factor bound from the other. The vocabulary the two share lives with `NumpyTimeDataType` in `_families` now, and both still export it. `tests/v3/test_extension_points.py` and `test_shape_properties.py` were two modules about one thing, `Context.resolve` -- the second with a docstring citing a test module and three names that no longer exist. They are one module, `test_resolve.py`, and the empty directory the rule registry's deletion left behind is gone. No behaviour changes; over 40,000 documents nothing differs. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_entity.py | 8 +- .../src/zarr_metadata/v3/_extension_points.py | 34 ----- .../src/zarr_metadata/v3/_registry.py | 13 +- .../zarr_metadata/v3/data_type/_families.py | 40 +++++- .../v3/data_type/numpy_datetime64.py | 10 +- .../v3/data_type/numpy_timedelta64.py | 34 +---- .../zarr-metadata/tests/v3/test_resolve.py | 123 ++++++++++++++++++ 7 files changed, 179 insertions(+), 83 deletions(-) delete mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py create mode 100644 packages/zarr-metadata/tests/v3/test_resolve.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 960a54ee74..994eba5ed3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -109,10 +109,10 @@ ] """The v3 array metadata fields whose values name an extension. -Here rather than in `_extension_points` because an entity that contains -other entities has to say which point it is reading them at, and -`_extension_points` also folds `r` names -- which means importing the -data types, which import this. +Names are unique only within a point -- `bytes` is both a core codec and +a registered data type -- so every table in this package is keyed by +point and then by name, and an entity that contains other entities says +which point it reads them at. """ # Left to infer their `Literal` types rather than widened to diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py deleted file mode 100644 index eea1b6542d..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py +++ /dev/null @@ -1,34 +0,0 @@ -"""The Zarr v3 extension points. - -Names are unique only within an extension point (`bytes` is both a core -codec and a registered data type), so every table in this package is -keyed by `(field, name)`. - -A name that no key matches may still belong to a *family* -- one class -covering many spellings, as the raw-byte data types cover every `r`. -`Context.resolve` asks each entity through `accepts`, so a family is -registered like anything else and this module holds no table of -spellings. -""" - -from __future__ import annotations - -from zarr_metadata.v3._entity import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, - STORAGE_TRANSFORMERS, - ExtensionPointField, -) -from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY - -__all__ = [ - "CHUNK_GRID", - "CHUNK_KEY_ENCODING", - "CODECS", - "DATA_TYPE", - "RAW_BYTES_FAMILY", - "STORAGE_TRANSFORMERS", - "ExtensionPointField", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 33e57c5d33..604557df5f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -31,6 +31,10 @@ validate_metadata_field_v3, ) from zarr_metadata.v3._entity import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, + CODECS, + DATA_TYPE, STORAGE_TRANSFORMERS, ChunkGridEntity, CodecEntity, @@ -39,12 +43,6 @@ Opaque, named_configuration, ) -from zarr_metadata.v3._extension_points import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, -) from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid from zarr_metadata.v3.chunk_key_encoding.default import DefaultChunkKeyEncoding @@ -82,8 +80,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from zarr_metadata.v3._entity import Loc - from zarr_metadata.v3._extension_points import ExtensionPointField + from zarr_metadata.v3._entity import ExtensionPointField, Loc class EntityTables(TypedDict): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 94205d71e7..a207141ef3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -15,7 +15,7 @@ from collections.abc import Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Final +from typing import TYPE_CHECKING, ClassVar, Final, Literal from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( @@ -138,9 +138,42 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP ) +NumpyTimeUnit = Literal[ + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" +] +"""Time unit codes shared by `numpy.datetime64` and `numpy.timedelta64`.""" + +NUMPY_TIME_UNIT: Final = ( + "Y", + "M", + "W", + "D", + "h", + "m", + "s", + "ms", + "us", + "μs", + "ns", + "ps", + "fs", + "as", + "generic", +) +"""Tuple of the permitted `unit` values, in numpy's order from coarse to fine.""" + +NUMPY_TIME_MAX_SCALE_FACTOR: Final = 2**31 - 1 +"""The largest `scale_factor` numpy stores: the field is a signed int32.""" + + @dataclass(frozen=True) class NumpyTimeDataType(DataTypeEntity, base=True): - """A numpy time scalar: a signed 64-bit count of units, or `NaT`.""" + """A numpy time scalar: a signed 64-bit count of units, or `NaT`. + + The vocabulary the two time types share -- the unit codes and the + scale-factor bound -- lives here with the family, so neither sibling + imports it from the other. + """ scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -158,10 +191,13 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP __all__ = [ "FLOAT_SPECIALS", + "NUMPY_TIME_MAX_SCALE_FACTOR", + "NUMPY_TIME_UNIT", "ComplexDataType", "FloatDataType", "IntegerDataType", "NumpyTimeDataType", + "NumpyTimeUnit", "as_sequence", "byte_values", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 223f8c7f26..26ce5a67ba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -13,9 +13,10 @@ Interval, StorageClass, ) -from zarr_metadata.v3.data_type._families import NumpyTimeDataType -from zarr_metadata.v3.data_type.numpy_timedelta64 import ( +from zarr_metadata.v3.data_type._families import ( NUMPY_TIME_MAX_SCALE_FACTOR, + NumpyTimeDataType, + NumpyTimeUnit, ) NUMPY_DATETIME64_DATA_TYPE_NAME: Final = "numpy.datetime64" @@ -24,11 +25,6 @@ NumpyDatetime64DataTypeName = Literal["numpy.datetime64"] """Literal type of the `name` field of the `numpy.datetime64` data type.""" -NumpyTimeUnit = Literal[ - "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" -] -"""Time unit codes used by numpy.datetime64.""" - class NumpyDatetime64Configuration(TypedDict, closed=True): """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 40698f9c75..19d3536c49 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -13,7 +13,12 @@ Interval, StorageClass, ) -from zarr_metadata.v3.data_type._families import NumpyTimeDataType +from zarr_metadata.v3.data_type._families import ( + NUMPY_TIME_MAX_SCALE_FACTOR, + NUMPY_TIME_UNIT, + NumpyTimeDataType, + NumpyTimeUnit, +) NUMPY_TIMEDELTA64_DATA_TYPE_NAME: Final = "numpy.timedelta64" """The `name` field value of the `numpy.timedelta64` data type.""" @@ -21,33 +26,6 @@ NumpyTimedelta64DataTypeName = Literal["numpy.timedelta64"] """Literal type of the `name` field of the `numpy.timedelta64` data type.""" -NumpyTimeUnit = Literal[ - "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as", "generic" -] -"""Time unit codes used by numpy.timedelta64.""" - -NUMPY_TIME_MAX_SCALE_FACTOR: Final = 2**31 - 1 -"""The largest `scale_factor` numpy stores: the field is a signed int32.""" - -NUMPY_TIME_UNIT: Final = ( - "Y", - "M", - "W", - "D", - "h", - "m", - "s", - "ms", - "us", - "μs", - "ns", - "ps", - "fs", - "as", - "generic", -) -"""Runtime tuple of the permitted `numpy.timedelta64`/`numpy.datetime64` unit strings.""" - class NumpyTimedelta64Configuration(TypedDict, closed=True): """ diff --git a/packages/zarr-metadata/tests/v3/test_resolve.py b/packages/zarr-metadata/tests/v3/test_resolve.py new file mode 100644 index 0000000000..3d9755235e --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_resolve.py @@ -0,0 +1,123 @@ +"""How a name reaches the entity that answers for it. + +`Context.resolve` tries the name as a key and, when nothing is keyed by +it, asks each entity at that point whether the name is one of its own -- +which is how a family such as `r` covers an unbounded set of names +from one registration. The examples pin the cases that matter; the +properties cover the family, which no example-based test can sample. +""" + +from __future__ import annotations + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr_metadata.rules import validate_array_metadata_v3 +from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.data_type.numpy_datetime64 import NumpyDatetime64DataType +from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY, RawBytesDataType +from zarr_metadata.v3.data_type.uint8 import Uint8DataType +from zarr_metadata.v3.entity import ( + CHUNK_GRID, + CODECS, + CORE_AND_EXTENSIONS, + DATA_TYPE, + MetadataEntity, +) + +# (field, name, the entity that answers for it — None when nothing does) +RESOLUTIONS: dict[str, tuple[str, str, type[MetadataEntity] | None]] = { + "plain-dtype": (DATA_TYPE, "uint8", Uint8DataType), + "dotted-dtype": (DATA_TYPE, "numpy.datetime64", NumpyDatetime64DataType), + "raw-8": (DATA_TYPE, "r8", RawBytesDataType), + "raw-24": (DATA_TYPE, "r24", RawBytesDataType), + # A malformed member reaches the family too: a misspelling of + # something we model must be reported as such, not pass as an unknown + # third-party extension. + "raw-not-multiple-of-8": (DATA_TYPE, "r12", RawBytesDataType), + "raw-zero": (DATA_TYPE, "r0", RawBytesDataType), + # Tables are per point, so the family cannot be reached from another. + "raw-shaped-codec-name": (CODECS, "r8", None), + "codec": (CODECS, "blosc", BloscCodec), + "unknown": (CODECS, "zfpy", None), + # The family's key is invented, so no document may write it. + "the-family-key-itself": (DATA_TYPE, RAW_BYTES_FAMILY, None), +} + + +@pytest.mark.parametrize(("field", "name", "expected"), RESOLUTIONS.values(), ids=list(RESOLUTIONS)) +def test_a_name_resolves_to_the_entity_that_answers_for_it( + field: str, name: str, expected: type[MetadataEntity] | None +) -> None: + assert CORE_AND_EXTENSIONS.resolve(field, name) is expected # type: ignore[arg-type] + + +@given(width=st.integers(min_value=0, max_value=2**32)) +def test_every_numeric_r_spelling_resolves_to_the_family(width: int) -> None: + # Including malformed widths (0, 12, anything not a multiple of 8): + # the family claims a name by grammar shape, not by validity, so a + # misspelled member of a family we model is reported as a misspelling + # rather than passing as an unknown third-party extension. + assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, f"r{width}") is RawBytesDataType + + +@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from([CODECS, CHUNK_GRID])) +def test_r_shaped_names_resolve_to_nothing_outside_data_types(width: int, field: str) -> None: + # The family belongs to `data_type`; a codec that happens to be named + # `r8` must not reach it. + assert CORE_AND_EXTENSIONS.resolve(field, f"r{width}") is None # type: ignore[arg-type] + + +# The scan `resolve` falls back to asks every entity, so a name no entity +# claims has to come back as nothing however many are registered. +_UNCLAIMED = st.text(min_size=1).filter( + lambda name: ( + not (name.startswith("r") and name[1:].isdigit()) + and name not in CORE_AND_EXTENSIONS.entities["data_type"] + ) +) + + +@given(name=_UNCLAIMED) +def test_a_name_no_entity_claims_resolves_to_nothing(name: str) -> None: + assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, name) is None + + +def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: + # Zarr identifiers are registry-allocated. A private codec named + # `bytes` has left the compatibility contract, and saying so is the + # correct answer rather than a limitation, so nothing here defends + # against collisions. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ({"name": "bytes", "configuration": {"width": 7}},), + } + problems = validate_array_metadata_v3(document) + assert [(p.loc, p.kind) for p in problems] == [ + (("codecs", 0, "configuration", "width"), "unknown_key") + ] + + +def test_forging_the_family_sentinel_cannot_change_a_verdict() -> None: + # A literal "r" data type mislabels nothing: the family claims its + # names through `accepts`, not through the table key, so no validation + # verdict depends on the sentinel being unforgeable. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": RAW_BYTES_FAMILY, + "fill_value": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), + } + # Unjudged as an unknown data type, exactly as any unmodelled name is. + assert validate_array_metadata_v3(document) == () From 6975236c99d98495193ed633ec22133a24550837 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 11:21:00 +0200 Subject: [PATCH 065/107] refactor(zarr-metadata): the document owns its canonical form; rules is a door `rules/_canonical.py` read the document twice -- once to validate, once to get an object it could walk -- and then walked four of the document's five entity fields by hand, never reaching `storage_transformers`. It was the last hand-written recursion in the package, and it lived in the facade because the document had no `canonical` to put it on. `ArrayDocumentV3.canonical()` walks the fields that hold an entity the way every entity containing entities does, and applies the one rule that is the document's own; `to_json()` writes the document back with every entity in its JSON form. `canonicalize_array_metadata_v3` reads once and reports exactly the problems it did before, structural and semantic together on any mapping. The group's consolidated-metadata recursion moves beside the array's semantics in `v3/_document.py` -- `pydantic.py` was already importing it as a private module, which is what public-in-practice looks like -- and the one v2 rule to `v2/_document.py`. `zarr_metadata.rules` keeps every name it exported and becomes what it was described as: the door, with the scope it asks in. The four walkers `_document` now shares with `_entity` lose their underscores, as `within` and `coerce_members` never had them. Over 40,000 documents nothing differs; on 3.14 and 3.11 alike. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.8.md | 8 ++ packages/zarr-metadata/changes/4379.misc.1.md | 6 + .../src/zarr_metadata/pydantic.py | 2 +- .../src/zarr_metadata/rules/__init__.py | 4 +- .../src/zarr_metadata/rules/_canonical.py | 114 ------------------ .../src/zarr_metadata/rules/_documents.py | 81 +++++++++++-- .../src/zarr_metadata/rules/_v3_group.py | 88 -------------- .../{rules/_v2_array.py => v2/_document.py} | 2 +- .../src/zarr_metadata/v3/_document.py | 113 ++++++++++++++++- .../src/zarr_metadata/v3/_entity.py | 54 ++++----- .../tests/rules/test_canonical.py | 8 +- .../zarr-metadata/tests/v3/test_entities.py | 63 +++++++++- 12 files changed, 295 insertions(+), 248 deletions(-) delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py rename packages/zarr-metadata/src/zarr_metadata/{rules/_v2_array.py => v2/_document.py} (95%) diff --git a/packages/zarr-metadata/changes/4379.feature.8.md b/packages/zarr-metadata/changes/4379.feature.8.md index 3e0cee0d27..0df75e570e 100644 --- a/packages/zarr-metadata/changes/4379.feature.8.md +++ b/packages/zarr-metadata/changes/4379.feature.8.md @@ -29,3 +29,11 @@ either would pin a grid that currently adapts. Two properties are asserted over generated documents: canonicalizing twice changes nothing further, and canonicalizing never changes a verdict. + +The document owns its canonical form. `ArrayDocumentV3.canonical()` +walks the fields that hold an entity the way every entity containing +entities does -- `storage_transformers` included, which the hand-written +walk it replaces never reached -- and applies the one rule that is the +document's own. `canonicalize_array_metadata_v3` is the door onto it, +and reads the document once: the entities that judge it are the entities +that are rewritten. diff --git a/packages/zarr-metadata/changes/4379.misc.1.md b/packages/zarr-metadata/changes/4379.misc.1.md index 61efd43903..0425c9471b 100644 --- a/packages/zarr-metadata/changes/4379.misc.1.md +++ b/packages/zarr-metadata/changes/4379.misc.1.md @@ -23,3 +23,9 @@ its own `kind`. Removed with it: `codec_kind_of_name`, `ARRAY_ARRAY_CODEC_NAMES`, `ARRAY_BYTES_CODEC_NAMES`, `BYTES_BYTES_CODEC_NAMES`, and `codec.blosc.canonical_configuration`, whose one job is `BloscCodec.canonical()`. + +`zarr_metadata.rules` is a door and nothing else: `validate_*`, +`parse_*` and `canonicalize_*` with the scope they ask in. The logic +that had accumulated beside them -- a group's consolidated-metadata +recursion, the one v2 rule, the canonicalizer's walk -- lives with the +documents it is about, in `v3._document` and `v2._document`. diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index 11d27d0a7e..6488b47547 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -60,7 +60,7 @@ class ArrayManifest(BaseModel): ZarrV3MetadataFieldJSON as _ZarrV3MetadataFieldSchema, ) from zarr_metadata.model._validation import arrays_to_tuples, validate_consolidated_metadata_v3 -from zarr_metadata.rules._v3_group import consolidated_entries_problems +from zarr_metadata.v3._document import consolidated_entries_problems if TYPE_CHECKING: from collections.abc import Callable diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py index 9f3dd58e3b..597c5c8783 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/__init__.py @@ -14,12 +14,10 @@ round-trips preserve those unmodeled members. """ -from zarr_metadata.rules._canonical import ( +from zarr_metadata.rules._documents import ( Canonical, Invalid, canonicalize_array_metadata_v3, -) -from zarr_metadata.rules._documents import ( parse_array_metadata_v2, parse_array_metadata_v3, parse_group_metadata_v2, diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py b/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py deleted file mode 100644 index 10dc200c0e..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_canonical.py +++ /dev/null @@ -1,114 +0,0 @@ -"""One document in, one canonical document or one report of why not. - -`canonicalize_array_metadata_v3` takes a *syntactically* valid document -- -one the model layer has already accepted, so every member is present and -typed as its TypedDict declares -- and answers with -`Canonical[T] | Invalid`: either the same document in canonical form or -every reason it is not semantically valid. Testing the literal `valid` -field narrows to one or the other. - -Canonical means the simplest spelling with the same meaning, and each -entity decides that for itself in its own `canonical`: an entity whose -configuration carries nothing collapses to its bare name, `blosc` drops a -`typesize` that `shuffle` renders ignored, a rectilinear dimension's chunk -sizes run-length encode. This module only collects the answers, and the -fields no entity owns -- `dimension_names` of nothing but nulls says what -omitting the field says. - -Two properties are worth holding on to, and -`tests/rules/test_canonical.py` asserts both: canonicalizing twice -changes nothing further, and canonicalizing never changes a verdict. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast - -from zarr_metadata.model._array import ZarrV3ArrayMetadata -from zarr_metadata.model._validation import arrays_to_tuples -from zarr_metadata.rules._documents import validate_array_metadata_v3 -from zarr_metadata.v3._document import read_array_v3 -from zarr_metadata.v3._entity import MetadataEntity -from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context - -if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.model._validation import ValidationProblem - from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON - -DocumentT = TypeVar("DocumentT") - - -@dataclass(frozen=True, slots=True) -class Canonical(Generic[DocumentT]): - """A semantically valid document, in its simplest equivalent spelling.""" - - document: DocumentT - valid: Literal[True] = True - - -@dataclass(frozen=True, slots=True) -class Invalid: - """Every reason a document is not semantically valid; never empty.""" - - problems: tuple[ValidationProblem, ...] - valid: Literal[False] = False - - def __post_init__(self) -> None: - if len(self.problems) == 0: - msg = "Invalid requires at least one validation problem" - raise ValueError(msg) - - -def _canonical_document(document: Mapping[str, object], context: Context) -> dict[str, object]: - """Each entity in its own canonical spelling, and the rest as written.""" - array, _ = read_array_v3(document, context) - out = dict(document) - for key in ("data_type", "chunk_grid", "chunk_key_encoding"): - entity = getattr(array, key) - if isinstance(entity, MetadataEntity): - out[key] = entity.canonical().to_json() - if "codecs" in out: - out["codecs"] = tuple( - codec.canonical().to_json() if isinstance(codec, MetadataEntity) else codec.json - for codec in array.codecs - ) - names = out.get("dimension_names") - if isinstance(names, tuple) and all( - entry is None for entry in cast("tuple[object, ...]", names) - ): - # Every dimension unnamed says what saying nothing says. - del out["dimension_names"] - return out - - -def canonicalize_array_metadata_v3( - document: ZarrV3ArrayMetadataJSON, *, context: Context = CORE_AND_EXTENSIONS -) -> Canonical[ZarrV3ArrayMetadataJSON] | Invalid: - """`document` in canonical form, or every reason it is not valid. - - Expects a document the model layer has already accepted. Passing one - it has not is not an error -- the semantic problems are reported the - same way -- but the structural problems come back too, and the result - is `Invalid` rather than a canonical document. - """ - normalized = cast("ZarrV3ArrayMetadataJSON", arrays_to_tuples(document)) - problems = validate_array_metadata_v3(normalized, context=context) - if len(problems) != 0: - return Invalid(problems) - # Normalized first, so a document spelled with JSON arrays reaches the - # same fixpoint as the tuple spelling. It did not: the per-field - # simplifications test for `tuple`, and the validator was normalizing - # on a copy the canonicalizer never saw. - canonical = _canonical_document(normalized, context) - # The model layer's round trip normalizes the fields no entity owns. - return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) - - -__all__ = [ - "Canonical", - "Invalid", - "canonicalize_array_metadata_v3", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index c753620167..e106091075 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -1,17 +1,25 @@ -"""Whole-document structural and composition validation. +"""Whole-document validation and canonicalization: the door. These `validate_*` and `parse_*` functions mirror the model API but apply -both validation layers. There is deliberately no `is_*` counterpart: -composition validity is stricter than TypedDict membership, so a guard -here could not narrow honestly. Use `zarr_metadata.model.is_*` for type -narrowing. +both validation layers, and `canonicalize_array_metadata_v3` answers with +`Canonical[T] | Invalid`: the document in its simplest equivalent +spelling, or every reason it is not valid. The work is done by the +documents themselves -- `zarr_metadata.v3._document` and +`zarr_metadata.v2._document` -- and what this module decides is which +entities are in scope while it asks. + +There is deliberately no `is_*` counterpart: composition validity is +stricter than TypedDict membership, so a guard here could not narrow +honestly. Use `zarr_metadata.model.is_*` for type narrowing. """ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, cast +from dataclasses import dataclass +from typing import TYPE_CHECKING, Generic, Literal, TypeVar, cast +from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._validation import ( MetadataValidationError, arrays_to_tuples, @@ -28,9 +36,8 @@ from zarr_metadata.model._validation import ( validate_group_metadata_v3 as _validate_group_structure_v3, ) -from zarr_metadata.rules._v2_array import array_problems_v2 -from zarr_metadata.rules._v3_group import group_problems_v3 -from zarr_metadata.v3._document import array_problems_v3 +from zarr_metadata.v2._document import array_problems_v2 +from zarr_metadata.v3._document import array_problems_v3, group_problems_v3, read_array_v3 from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context if TYPE_CHECKING: @@ -45,6 +52,29 @@ _StructuralValidator = Callable[[object], tuple[ValidationProblem, ...]] _SemanticValidator = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] +DocumentT = TypeVar("DocumentT") + + +@dataclass(frozen=True, slots=True) +class Canonical(Generic[DocumentT]): + """A semantically valid document, in its simplest equivalent spelling.""" + + document: DocumentT + valid: Literal[True] = True + + +@dataclass(frozen=True, slots=True) +class Invalid: + """Every reason a document is not semantically valid; never empty.""" + + problems: tuple[ValidationProblem, ...] + valid: Literal[False] = False + + def __post_init__(self) -> None: + if len(self.problems) == 0: + msg = "Invalid requires at least one validation problem" + raise ValueError(msg) + def _no_semantics(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: """v2 group documents carry no cross-field constraints.""" @@ -195,7 +225,40 @@ def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: return cast("ZarrV2GroupMetadataJSON", normalized) +def canonicalize_array_metadata_v3( + document: ZarrV3ArrayMetadataJSON, *, context: Context = CORE_AND_EXTENSIONS +) -> Canonical[ZarrV3ArrayMetadataJSON] | Invalid: + """`document` in canonical form, or every reason it is not valid. + + Canonical means the simplest spelling with the same meaning, and the + document decides that for itself in `ArrayDocumentV3.canonical`: each + entity in its own canonical form, and `dimension_names` of nothing + but nulls omitted. Two properties are worth holding on to, and + `tests/rules/test_canonical.py` asserts both: canonicalizing twice + changes nothing further, and canonicalizing never changes a verdict. + + Expects a document the model layer has already accepted. Passing one + it has not is not an error -- the semantic problems are reported the + same way -- but the structural problems come back too, and the result + is `Invalid` rather than a canonical document. The document is read + once: the entities that judge it are the entities that are rewritten. + """ + normalized = arrays_to_tuples(document) + problems = _validate_structure_v3(normalized) + if not isinstance(normalized, Mapping): + return Invalid(problems) + array, found = read_array_v3(cast("Mapping[str, object]", normalized), context) + problems = (*problems, *found, *array.problems()) + if len(problems) != 0: + return Invalid(problems) + canonical = cast("ZarrV3ArrayMetadataJSON", array.canonical().to_json()) + return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) + + __all__ = [ + "Canonical", + "Invalid", + "canonicalize_array_metadata_v3", "parse_array_metadata_v2", "parse_array_metadata_v3", "parse_group_metadata_v2", diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py b/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py deleted file mode 100644 index fa0f2c26b1..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v3_group.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Semantic checks for v3 group metadata documents. - -A group says almost nothing that can be wrong on its own. The one thing -it can carry is consolidated metadata -- the child documents of a whole -subtree, inline -- and each of those is judged exactly as it would be -standing alone, at its own path. - -`consolidated_metadata` is not a declared member of the group TypedDict: -the spec grandfathers it as a convention that "lacks the name member -required of extension objects". -""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import TYPE_CHECKING, cast - -from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.v3._document import array_problems_v3 -from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context - -if TYPE_CHECKING: - from collections.abc import Sequence - - from zarr_metadata.v3._entity import Loc - - -def _prefixed(loc: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: - """Re-base every problem's `loc` under `loc`, for a nested document.""" - return tuple( - ValidationProblem((*loc, *found.loc), found.message, found.kind) for found in problems - ) - - -def _as_string_mapping(value: object) -> Mapping[str, object] | None: - """`value` as a string-keyed mapping, or None if it is not one.""" - if not isinstance(value, Mapping): - return None - mapping = cast("Mapping[object, object]", value) - if any(not isinstance(key, str) for key in mapping): - return None - return cast("Mapping[str, object]", mapping) - - -def group_problems_v3( - document: Mapping[str, object], context: Context = CORE_AND_EXTENSIONS -) -> tuple[ValidationProblem, ...]: - """Every semantic problem in a v3 group document.""" - if "consolidated_metadata" not in document: - return () - return consolidated_entries_problems( - document["consolidated_metadata"], ("consolidated_metadata",), context - ) - - -def consolidated_entries_problems( - value: object, loc: Loc = (), context: Context = CORE_AND_EXTENSIONS -) -> tuple[ValidationProblem, ...]: - """Semantic problems in an inline consolidated envelope's children. - - Structural validity of the envelope and its entries is the model - layer's job; an entry that is not interpretable as a node document - declines in its favour. - """ - consolidated = _as_string_mapping(value) - if consolidated is None: - return () - metadata = _as_string_mapping(consolidated.get("metadata")) - if metadata is None: - return () - problems: list[ValidationProblem] = [] - for path, entry in metadata.items(): - node = _as_string_mapping(entry) - if node is None: - continue - entry_loc = (*loc, "metadata", path) - node_type = node.get("node_type") - if node_type == "array": - problems.extend(_prefixed(entry_loc, array_problems_v3(node, context))) - elif node_type == "group": - problems.extend(_prefixed(entry_loc, group_problems_v3(node, context))) - return tuple(problems) - - -__all__ = [ - "consolidated_entries_problems", - "group_problems_v3", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py b/packages/zarr-metadata/src/zarr_metadata/v2/_document.py similarity index 95% rename from packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py rename to packages/zarr-metadata/src/zarr_metadata/v2/_document.py index 376eae259e..05a62a1f4e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_v2_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_document.py @@ -1,4 +1,4 @@ -"""Semantic checks for v2 array metadata documents. +"""What a v2 array document can get wrong beyond its shape. Deliberately small: the one cross-field constraint the package interprets is that `chunks` and `shape` agree on dimensionality. v2 has no extension diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 12cda0ea8f..0c0cbb4fce 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -17,7 +17,7 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Final, cast from zarr_metadata.model._validation import ( @@ -41,6 +41,10 @@ ExtensionPointField, MetadataEntity, Opaque, + canonicalize_nested, + contains_entity, + field_hints, + render_nested, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid @@ -49,6 +53,8 @@ if TYPE_CHECKING: from collections.abc import Sequence + from zarr_metadata.v3._entity import Loc + @dataclass(frozen=True, slots=True) class ArrayDocumentV3: @@ -83,6 +89,43 @@ def problems(self) -> tuple[ValidationProblem, ...]: *chain_problems(self.codecs, self.parts, ("codecs",)), ) + def canonical(self) -> ArrayDocumentV3: + """This document in the simplest form that means the same thing. + + Each entity in its own canonical form -- the same walk over the + fields that hold one that every entity containing entities gets + -- and the one rule that is the document's own: `dimension_names` + of nothing but nulls says what omitting the field says. A + *transformation*, asked for by `canonicalize_array_metadata_v3`; + `to_json` does not apply it. + """ + entities = { + name: canonicalize_nested(annotation, getattr(self, name)) + for name, annotation in field_hints(type(self)).items() + if contains_entity(annotation) + } + document = dict(self.document) + names = document.get("dimension_names") + if isinstance(names, tuple) and all( + entry is None for entry in cast("tuple[object, ...]", names) + ): + del document["dimension_names"] + return replace(self, document=document, **entities) + + def to_json(self) -> dict[str, object]: + """The document as it would be written: every entity in its JSON form. + + Faithful to what was read, member for member; the fields that + are not extension points come back exactly as the document had + them. Ask `canonical` first for the simplest equivalent spelling. + """ + rendered = { + name: render_nested(annotation, getattr(self, name)) + for name, annotation in field_hints(type(self)).items() + if contains_entity(annotation) + } + return {**self.document, **rendered} + @classmethod def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> ArrayDocumentV3: """A v3 array document read into entities, or raise. @@ -231,8 +274,76 @@ def array_problems_v3( return (*problems, *array.problems()) +def _prefixed(loc: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: + """Re-base every problem's `loc` under `loc`, for a nested document.""" + return tuple( + ValidationProblem((*loc, *found.loc), found.message, found.kind) for found in problems + ) + + +def _as_string_mapping(value: object) -> Mapping[str, object] | None: + """`value` as a string-keyed mapping, or None if it is not one.""" + if not isinstance(value, Mapping): + return None + mapping = cast("Mapping[object, object]", value) + if any(not isinstance(key, str) for key in mapping): + return None + return cast("Mapping[str, object]", mapping) + + +def group_problems_v3( + document: Mapping[str, object], context: Context = CORE_AND_EXTENSIONS +) -> tuple[ValidationProblem, ...]: + """Every semantic problem in a v3 group document. + + A group says almost nothing that can be wrong on its own. The one + thing it can carry is consolidated metadata -- the child documents of + a whole subtree, inline -- and each of those is judged exactly as it + would be standing alone, at its own path. `consolidated_metadata` is + not a declared member of the group TypedDict: the spec grandfathers + it as a convention that "lacks the name member required of extension + objects". + """ + if "consolidated_metadata" not in document: + return () + return consolidated_entries_problems( + document["consolidated_metadata"], ("consolidated_metadata",), context + ) + + +def consolidated_entries_problems( + value: object, loc: Loc = (), context: Context = CORE_AND_EXTENSIONS +) -> tuple[ValidationProblem, ...]: + """Semantic problems in an inline consolidated envelope's children. + + Structural validity of the envelope and its entries is the model + layer's job; an entry that is not interpretable as a node document + declines in its favour. + """ + consolidated = _as_string_mapping(value) + if consolidated is None: + return () + metadata = _as_string_mapping(consolidated.get("metadata")) + if metadata is None: + return () + problems: list[ValidationProblem] = [] + for path, entry in metadata.items(): + node = _as_string_mapping(entry) + if node is None: + continue + entry_loc = (*loc, "metadata", path) + node_type = node.get("node_type") + if node_type == "array": + problems.extend(_prefixed(entry_loc, array_problems_v3(node, context))) + elif node_type == "group": + problems.extend(_prefixed(entry_loc, group_problems_v3(node, context))) + return tuple(problems) + + __all__ = [ "ArrayDocumentV3", "array_problems_v3", + "consolidated_entries_problems", + "group_problems_v3", "read_array_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 994eba5ed3..a4deaf80a9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -386,7 +386,7 @@ class variables are skipped by text before anything is evaluated. return dict(vars(klass).get("__annotations__", {})) -def _field_hints(cls: type) -> dict[str, object]: +def field_hints(cls: type) -> dict[str, object]: """The dataclass fields of `cls`, resolved, base first. Each class's own annotations are resolved in that class's module, @@ -710,7 +710,7 @@ def _compile_typeddict(inner: object) -> TypeCheck | None: def _compile_record(inner: object) -> TypeCheck | None: if not isinstance(inner, type): # pragma: no cover - the predicate says it is return None - members = _members_of(_field_hints(inner)) + members = _members_of(field_hints(inner)) return None if members is None else mapping_of(members) @@ -748,7 +748,7 @@ def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], l """ derived: dict[str, tuple[bool, TypeCheck]] = {} unread: list[str] = [] - for name, annotation in _field_hints(cls).items(): + for name, annotation in field_hints(cls).items(): inner, metadata = _strip(annotation) if any(entry is FROM_NAME for entry in metadata): continue @@ -906,7 +906,7 @@ def per_position(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: elif isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): members = { name: check - for name, field_annotation in _field_hints(inner).items() + for name, field_annotation in field_hints(inner).items() if (check := value_check_for(field_annotation)) is not None } if len(members) != 0: @@ -934,22 +934,22 @@ def both(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return both -def _contains_entity(annotation: object) -> bool: +def contains_entity(annotation: object) -> bool: """Whether a value of this type holds a nested metadata field anywhere in it.""" inner, _ = _strip(annotation) if _is_entity_type(inner): return True origin = get_origin(inner) if _is_union(inner): - return any(_contains_entity(arg) for arg in get_args(inner) if arg is not UNSET) + return any(contains_entity(arg) for arg in get_args(inner) if arg is not UNSET) if origin is tuple: - return any(_contains_entity(arg) for arg in get_args(inner) if arg is not Ellipsis) + return any(contains_entity(arg) for arg in get_args(inner) if arg is not Ellipsis) if is_typeddict(inner): return any( - _contains_entity(value) for value in get_type_hints(inner, include_extras=True).values() + contains_entity(value) for value in get_type_hints(inner, include_extras=True).values() ) if isinstance(inner, type) and is_dataclass(inner): - return any(_contains_entity(value) for value in _field_hints(inner).values()) + return any(contains_entity(value) for value in field_hints(inner).values()) return False @@ -978,7 +978,7 @@ def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: if origin is tuple: return [kind for arg in arguments if arg is not Ellipsis for kind in _entity_kinds(arg)] if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): - return [kind for value in _field_hints(inner).values() for kind in _entity_kinds(value)] + return [kind for value in field_hints(inner).values() for kind in _entity_kinds(value)] return [] @@ -1007,7 +1007,7 @@ def _element_annotations(inner: object, count: int) -> list[object]: def _fitting_branch(inner: object, value: object) -> object | None: """The branch of a union that holds an entity and whose shape `value` has.""" for branch in get_args(inner): - if branch is UNSET or not _contains_entity(branch): + if branch is UNSET or not contains_entity(branch): continue if _has_shape(_shape(branch), value): return branch @@ -1047,7 +1047,7 @@ def _resolve( entries = cast("Mapping[str, object]", value) members: dict[str, object] = {} found = [] - for name, field_annotation in _field_hints(inner).items(): + for name, field_annotation in field_hints(inner).items(): if name not in entries: continue member, problems = _resolve(field_annotation, entries[name], context, (*loc, name)) @@ -1057,7 +1057,7 @@ def _resolve( return value, () -def _render(annotation: object, value: object) -> object: +def render_nested(annotation: object, value: object) -> object: """`value` as a document would write it: every nested entity in its JSON form.""" if isinstance(value, MetadataEntity): return value.to_json() @@ -1066,25 +1066,25 @@ def _render(annotation: object, value: object) -> object: inner, _ = _strip(annotation) if _is_union(inner): branch = _fitting_branch(inner, value) - return value if branch is None else _render(branch, value) + return value if branch is None else render_nested(branch, value) if get_origin(inner) is tuple: entries = cast("tuple[object, ...]", value) return tuple( - _render(element, entry) + render_nested(element, entry) for element, entry in zip( _element_annotations(inner, len(entries)), entries, strict=True ) ) if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): return { - name: _render(field_annotation, getattr(value, name)) - for name, field_annotation in _field_hints(inner).items() + name: render_nested(field_annotation, getattr(value, name)) + for name, field_annotation in field_hints(inner).items() if getattr(value, name) is not UNSET } return value -def _canonicalize(annotation: object, value: object) -> object: +def canonicalize_nested(annotation: object, value: object) -> object: """`value` with every nested entity in its own canonical form.""" if isinstance(value, MetadataEntity): return value.canonical() @@ -1093,11 +1093,11 @@ def _canonicalize(annotation: object, value: object) -> object: inner, _ = _strip(annotation) if _is_union(inner): branch = _fitting_branch(inner, value) - return value if branch is None else _canonicalize(branch, value) + return value if branch is None else canonicalize_nested(branch, value) if get_origin(inner) is tuple: entries = cast("tuple[object, ...]", value) return tuple( - _canonicalize(element, entry) + canonicalize_nested(element, entry) for element, entry in zip( _element_annotations(inner, len(entries)), entries, strict=True ) @@ -1112,8 +1112,8 @@ def _canonicalize(annotation: object, value: object) -> object: return replace( value, **{ - name: _canonicalize(field_annotation, getattr(value, name)) - for name, field_annotation in _field_hints(inner).items() + name: canonicalize_nested(field_annotation, getattr(value, name)) + for name, field_annotation in field_hints(inner).items() }, ) return value @@ -1351,7 +1351,7 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: f"{', '.join(unsupported)}; declare one in `member_types`" ) raise TypeError(msg) - optional = {name: is_optional(annotation) for name, annotation in _field_hints(cls).items()} + optional = {name: is_optional(annotation) for name, annotation in field_hints(cls).items()} misstated = sorted( member for member, (required, _) in declared.items() @@ -1369,9 +1369,9 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: raise TypeError(msg) cls.member_types = {**derived, **declared} cls.configuration_required = any(required for required, _ in cls.member_types.values()) - hints = _field_hints(cls) + hints = field_hints(cls) cls.nested_members = { - name: annotation for name, annotation in hints.items() if _contains_entity(annotation) + name: annotation for name, annotation in hints.items() if contains_entity(annotation) } cls.value_checks = { name: check @@ -1551,7 +1551,7 @@ def canonical(self) -> Self: return replace( self, **{ - name: _canonicalize(annotation, getattr(self, name)) + name: canonicalize_nested(annotation, getattr(self, name)) for name, annotation in nested.items() }, ) @@ -1578,7 +1578,7 @@ def configuration(self) -> dict[str, object]: members = self._configuration_members() for name, annotation in type(self).nested_members.items(): if name in members: - members[name] = _render(annotation, members[name]) + members[name] = render_nested(annotation, members[name]) return deepcopy(members) value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) diff --git a/packages/zarr-metadata/tests/rules/test_canonical.py b/packages/zarr-metadata/tests/rules/test_canonical.py index ef7ed4dd61..d5da358af1 100644 --- a/packages/zarr-metadata/tests/rules/test_canonical.py +++ b/packages/zarr-metadata/tests/rules/test_canonical.py @@ -14,8 +14,12 @@ from hypothesis import HealthCheck, given, settings from tests.rules.strategies import valid_documents -from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.rules._canonical import Canonical, Invalid, canonicalize_array_metadata_v3 +from zarr_metadata.rules import ( + Canonical, + Invalid, + canonicalize_array_metadata_v3, + validate_array_metadata_v3, +) if TYPE_CHECKING: from collections.abc import Mapping diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 01c6164ec5..d05b45dcd6 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,11 +11,11 @@ import copy import dataclasses -from typing import Any, cast, get_args, get_type_hints +from typing import Any, ClassVar, Self, cast, get_args, get_type_hints import pytest -from zarr_metadata.model import MetadataValidationError +from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( @@ -579,3 +579,62 @@ def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: assert (("codecs", 1, "configuration", "typo_key"), "unknown_key") in { (problem.loc, problem.kind) for problem in caught.value.problems } + + +# A storage transformer: the one extension point nothing in the package +# models, so the only way to reach it is to register one. +@dataclasses.dataclass(frozen=True) +class AcmeShardCache(MetadataEntity): + """A third-party storage transformer with a member canonical form drops.""" + + verbose: bool | UNSET = UNSET + + identifier: ClassVar[str] = "acme.shard_cache" + + def canonical(self) -> Self: + return dataclasses.replace(super().canonical(), verbose=UNSET) + + +def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> None: + # `to_json` is faithful, entities included; `canonical` walks every + # field that holds an entity -- `storage_transformers` among them, + # which the hand-written walk it replaces never reached. + scope = CORE_AND_EXTENSIONS.extended_with( + storage_transformers={AcmeShardCache.identifier: AcmeShardCache} + ) + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,)}}, + "chunk_key_encoding": "default", + "codecs": ( + "bytes", + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "noshuffle", + "typesize": 4, + "blocksize": 0, + }, + }, + ), + "storage_transformers": ({"name": "acme.shard_cache", "configuration": {"verbose": True}},), + "dimension_names": (None,), + } + array = ArrayDocumentV3.from_json(document, context=scope) + assert array.to_json() == document + canonical = array.canonical().to_json() + assert canonical["codecs"] == ( + "bytes", + { + "name": "blosc", + "configuration": {"cname": "zstd", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0}, + }, + ) + assert canonical["storage_transformers"] == ("acme.shard_cache",) + assert "dimension_names" not in canonical From 799cea32d2c8391bd33cab90a369964bd203c6e5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 11:28:03 +0200 Subject: [PATCH 066/107] refactor(zarr-metadata): the base module is three modules `_entity.py` had reached 1,900 lines and was three things: the member checks every entity needs, the compiler that reads an entity's schema off its field annotations, and the entities themselves. `_checks.py` is the first -- one check is a function of a value and its location, `coerce_members` walks a configuration with a table of them. `_compile.py` is the second: `check_for` and its registry of shapes, the bound vocabulary and `value_check_for`, `validates`, and `derive_member_types`. It knows nothing of entities. A field typed as one is a shape like any other, and `_entity.py` registers it through `register_check` with the JSON shape and the description the compiler needs of it -- the same door a third party's shape comes in by, and the reason the split has no cycle. `_entity.py` keeps the classes, the guards and the walkers, and goes on importing every name its subclasses and the public door use, so no other module changes. The helpers that now cross a module boundary lose their underscores (`strip_annotation`, `field_hints`, `shape_of`, ...). The module docstring said `problems` was value-space and that document-level logic lived in `rules`; neither had been true for a while. Over 40,000 documents nothing differs; on 3.14 and 3.11 alike. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_checks.py | 253 ++++ .../src/zarr_metadata/v3/_compile.py | 787 +++++++++++++ .../src/zarr_metadata/v3/_document.py | 2 +- .../src/zarr_metadata/v3/_entity.py | 1028 ++--------------- .../tests/v3/test_extension_api.py | 4 +- 5 files changed, 1153 insertions(+), 921 deletions(-) create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_checks.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_compile.py diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py new file mode 100644 index 0000000000..a2ee3cc233 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py @@ -0,0 +1,253 @@ +"""The member checks every entity needs, and the walk that applies them. + +One check is a function of a value and its location that returns the +problems it found -- none, for a value of the right type. The scalars, +a closed set of names, a homogeneous sequence, and a nested metadata +field cover what a configuration member can be; `coerce_members` walks +a configuration with a table of them, distinguishing an unknown key, an +optional member that failed, and a required one that did. `within` and +`named_configuration` are how an entity's problems and envelope are read +from the document that holds it. +""" + +from __future__ import annotations + +# Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below +# (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class +# creation, and a name that exists only for the type checker is a NameError +# then -- for this package and for any tool introspecting an entity. +from collections.abc import Callable, Mapping, Sequence +from typing import ( + TYPE_CHECKING, + TypeAlias, + cast, +) + +from typing_extensions import TypeIs + +from zarr_metadata.model._validation import ( + ValidationProblem, + is_json, +) + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ProblemKind + + +Loc: TypeAlias = "tuple[str | int, ...]" + + +TypeCheck: TypeAlias = "Callable[[object, Loc], tuple[ValidationProblem, ...]]" +"""Whether one value has the type a member declares, and where if not.""" + + +MemberTypes: TypeAlias = "Mapping[str, tuple[bool, TypeCheck]]" +"""Per configuration member: whether it is required, and its type check.""" + + +def problem( + loc: Loc, message: str, kind: ProblemKind = "invalid_type" +) -> tuple[ValidationProblem, ...]: + """One problem, as the tuple every check returns.""" + return (ValidationProblem(loc, message, kind),) + + +def is_integer(value: object) -> TypeIs[int]: + """A JSON integer: an `int`, and not a `bool`. + + `True` is an `int` in Python and `true` is not a number in JSON, so + the two have to be told apart everywhere a number is expected. + """ + return not isinstance(value, bool) and isinstance(value, int) + + +def is_int(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """An integer, and not a bool -- JSON `true` is not the integer 1.""" + if not is_integer(value): + return problem(loc, f"expected an integer, got {value!r}") + return () + + +def is_str(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, str): + return problem(loc, f"expected a string, got {value!r}") + return () + + +def is_bool(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, bool): + return problem(loc, f"expected a boolean, got {value!r}") + return () + + +def is_json_value(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """Any JSON value at all -- the widest type a member can declare.""" + if not is_json(value): + return problem(loc, f"expected a JSON value, got {value!r}") + return () + + +def one_of(allowed: tuple[str, ...]) -> TypeCheck: + """A member whose type is a closed set of names.""" + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if value not in allowed: + return problem(loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value") + return () + + return check + + +def sequence_of(element: TypeCheck) -> TypeCheck: + """A member whose type is a sequence, checked element by element.""" + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, (list, tuple)): + return problem(loc, f"expected a sequence, got {value!r}") + elements: tuple[object, ...] = tuple(cast("list[object] | tuple[object, ...]", value)) + return tuple( + found for index, entry in enumerate(elements) for found in element(entry, (*loc, index)) + ) + + return check + + +def _as_tuples(value: object) -> object: + """Every JSON array in `value`, at any depth, as a tuple. + + The TypedDicts spell a JSON array as a tuple throughout, so a member + taken straight from parsed JSON would otherwise hold a list where its + own type says tuple -- and two documents differing only in that would + compare unequal. + """ + if isinstance(value, (list, tuple)): + entries = cast("list[object] | tuple[object, ...]", value) + return tuple(_as_tuples(entry) for entry in entries) + if isinstance(value, Mapping): + entries = cast("Mapping[str, object]", value) + return {key: _as_tuples(entry) for key, entry in entries.items()} + return value + + +def coerce_members( + configuration: Mapping[str, object], types: MemberTypes +) -> tuple[dict[str, object], tuple[ValidationProblem, ...], frozenset[str]]: + """The members `types` declares, taken from `configuration`. + + Returns what was accepted, every problem found, and the names of the + required members that could not be read. Three kinds of problem, and + they differ in that last part: + + - a key the entity does not declare says the value carries something + extra, not that it is wrong; + - an *optional* member of the wrong type leaves that member absent, + and everything else about the entity is still readable -- a bad + `index_location` says nothing about whether a shard's pipelines + are well formed, and silencing them would lose a real judgment; + - a *required* member missing or of the wrong type does stop it. + There is no honest reading of a `blosc` whose level is a string. + """ + problems: list[ValidationProblem] = [] + members: dict[str, object] = {} + unreadable: set[str] = set() + for key in configuration: + if key not in types: + problems.extend( + problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") + ) + for key, (required, check) in types.items(): + if key not in configuration: + if required: + problems.extend( + problem(("configuration", key), f"missing required key {key!r}", "missing_key") + ) + unreadable.add(key) + continue + # Normalized before the check, so a check only ever sees the tuples + # the TypedDicts declare -- never the lists raw JSON arrives as. + value = _as_tuples(configuration[key]) + found = check(value, ("configuration", key)) + problems.extend(found) + # An unknown key says the value carries something extra, not that + # it is the wrong type -- so the member is still readable, and + # dropping it here would make `to_json` lose what was written. + if all(entry.kind == "unknown_key" for entry in found): + members[key] = value + elif required: + unreadable.add(key) + return members, tuple(problems), frozenset(unreadable) + + +def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """A nested metadata field: a bare name or a named-configuration object. + + Only the envelope's shape. Which entity the name denotes, and whether + its configuration is well formed, is settled when the containing + entity reads it in scope. + """ + if not isinstance(value, (str, Mapping)): + return problem(loc, f"expected a metadata field, got {value!r}") + return () + + +def within(prefix: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: + """One entity's problems, located in the document that holds it. + + An entity reports relative to its own `configuration`, so that is what + goes between the field and the member. A problem with an empty + location is about the entity itself -- a malformed `r` name, a + codec that cannot encode what reaches it -- and lands on the field. + """ + return tuple( + ValidationProblem( + (*prefix, *(("configuration", *found.loc) if len(found.loc) != 0 else ())), + found.message, + found.kind, + ) + for found in problems + ) + + +def named_configuration( + value: object, +) -> tuple[str | None, Mapping[str, object] | None, bool]: + """Split metadata into `(name, configuration, must_understand)`. + + The shared shape every entity arrives in: a bare name, or an object + carrying one. A `None` name means the value is not a metadata field at + all; a `None` configuration means the bare spelling was used. + """ + if isinstance(value, str): + return value, None, True + if not isinstance(value, Mapping): + return None, None, True + entry = cast("Mapping[str, object]", value) + name = entry.get("name") + if not isinstance(name, str): + return None, None, True + configuration = entry.get("configuration") + must_understand = entry.get("must_understand", True) + return ( + name, + cast("Mapping[str, object]", configuration) if isinstance(configuration, Mapping) else None, + must_understand if isinstance(must_understand, bool) else True, + ) + + +__all__ = [ + "Loc", + "MemberTypes", + "TypeCheck", + "coerce_members", + "is_bool", + "is_int", + "is_integer", + "is_json_value", + "is_metadata_field", + "is_str", + "named_configuration", + "one_of", + "problem", + "sequence_of", + "within", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py new file mode 100644 index 0000000000..5f720cc91d --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py @@ -0,0 +1,787 @@ +"""What a field annotation says, read off it: the type, the bounds, the rules. + +An entity's dataclass fields are its schema, and this module is the +compiler over them. `check_for` turns an annotation into its type check +-- a scalar, a `Literal`, arrays homogeneous or fixed, unions, a nested +object described by a TypedDict or a record dataclass -- through a +registry of shapes that `register_check` keeps open, so a shape this +module does not know can be taught to it from outside. The bound +vocabulary (`Ge`, `Le`, `Interval`, ...) rides in `Annotated` and +`value_check_for` compiles it, at whatever depth it sits. `validates` +marks a rule about one member. `derive_member_types` is what an entity +reads its member table off. + +Nothing here knows what an entity is. A nested metadata field is a shape +like any other, registered by `_entity` through the same door, with the +JSON shape and the description the compiler needs of it. +""" + +from __future__ import annotations + +# Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below +# (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class +# creation, and a name that exists only for the type checker is a NameError +# then -- for this package and for any tool introspecting an entity. +import sys +import types +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, is_dataclass +from typing import ( + TYPE_CHECKING, + Annotated, + ClassVar, + Final, + Literal, + NotRequired, + Required, + TypeAlias, + TypeVar, + Union, + cast, + get_args, + get_origin, + get_type_hints, +) + +from typing_extensions import ReadOnly, is_typeddict + +from zarr_metadata._common import JSONValue +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.v3._checks import ( + is_bool, + is_int, + is_integer, + is_json_value, + is_str, + one_of, + problem, + sequence_of, +) + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ValidationProblem + from zarr_metadata.v3._checks import Loc, TypeCheck + + +class _FromName: + """The marker behind `FROM_NAME`.""" + + __slots__ = () + + def __repr__(self) -> str: + return "FROM_NAME" + + +FROM_NAME: Final = _FromName() +"""Marks a field carried by the metadata envelope's `name`, not its configuration. + + data_type_name: Annotated[str, FROM_NAME] + +A member all the same -- `value_problems` judges it -- but not a +configuration key, so it is neither read from nor written to a +`configuration` object. The raw-bytes family is the case: `r` keeps its +width in its name and has no configuration at all. +""" + + +@dataclass(frozen=True, slots=True) +class Ge: + """`Annotated[int, Ge(1)]`: the value is at least `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Gt: + """`Annotated[int, Gt(0)]`: the value is more than `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Le: + """`Annotated[int, Le(9)]`: the value is at most `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Lt: + """`Annotated[int, Lt(10)]`: the value is less than `bound`.""" + + bound: int | float + + +@dataclass(frozen=True, slots=True) +class Interval: + """`Annotated[int, Interval(ge=0, le=9)]`: the value lies within these bounds. + + These five are the `annotated_types` vocabulary -- what pydantic reads + and msgspec's `Meta` mirrors -- so a reader recognises them. Defined + here rather than imported, so the package keeps its one dependency. + A bound is a value rule: it runs only once the member has the type it + declared, at whatever depth the annotation puts it, so a bound on an + array's element type judges each element at its own position. + """ + + ge: int | float | None = None + gt: int | float | None = None + le: int | float | None = None + lt: int | float | None = None + + +def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: + """An annotation's type, and the metadata `Annotated` wrapped it in. + + `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict + key, not part of the value's type; peeled with the `Annotated` layers, + in whatever order they were written. + """ + metadata: list[object] = [] + while True: + origin = get_origin(annotation) + if origin is Annotated: + inner, *extras = get_args(annotation) + metadata.extend(extras) + annotation = inner + elif origin in (NotRequired, Required, ReadOnly): + (annotation,) = get_args(annotation) + else: + return annotation, tuple(metadata) + + +def own_annotations(klass: type) -> dict[str, object]: + """A class's own annotations, unevaluated. + + From 3.14 a class does not carry an `__annotations__` dict until it is + asked for one, and asking evaluates every annotation at once -- so a + `ClassVar` naming something imported only for the type checker would + fail the whole class. `annotationlib` can hand them back as the text + they were written as, which is what the callers here want anyway: + class variables are skipped by text before anything is evaluated. + Earlier versions leave the dict on the class, strings or values as + the module chose. + """ + if sys.version_info >= (3, 14): + import annotationlib + + return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) + return dict(vars(klass).get("__annotations__", {})) + + +def field_hints(cls: type) -> dict[str, object]: + """The dataclass fields of `cls`, resolved, base first. + + Each class's own annotations are resolved in that class's module, + and class variables are skipped *before* resolving, by text -- so a + `ClassVar` whose annotation names something imported only for the + type checker cannot fail class creation. `@dataclass` sees the same + set, in the same order. + """ + hints: dict[str, object] = {} + for ancestor in reversed(cls.__mro__): + raw = { + name: annotation + for name, annotation in own_annotations(ancestor).items() + if not is_class_var(annotation) + } + if len(raw) == 0: + continue + shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) + hints.update(get_type_hints(shell, include_extras=True)) + return hints + + +def is_union(annotation: object) -> bool: + return get_origin(annotation) in (Union, types.UnionType) + + +def is_optional(annotation: object) -> bool: + """Whether a field may be absent: its type admits `UNSET`.""" + inner, _ = strip_annotation(annotation) + return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) + + +def describe(annotation: object) -> str: + """The annotation as a message would name it: "an integer", "an object".""" + inner, _ = strip_annotation(annotation) + for registration in _CHECK_COMPILERS: + if registration.description is not None and registration.predicate(inner): + return registration.description + if inner is int: + return "an integer" + if inner is bool: + return "a boolean" + if inner is str: + return "a string" + if inner is JSONValue: + return "a JSON value" + origin = get_origin(inner) + if origin is Literal: + return f"one of {tuple(sorted(get_args(inner)))!r}" + if is_union(inner): + branches = [arg for arg in get_args(inner) if arg is not UNSET] + return " or ".join(describe(branch) for branch in branches) + if origin is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + return f"an array of {describe(arguments[0])} elements" + if len(arguments) == 2: + return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" + return f"an array of {len(arguments)} elements" + if is_typeddict(inner) or is_dataclass(inner): + return "an object" + return "a value" + + +def shape_of(annotation: object) -> str | None: + """The top-level JSON shape an annotation admits, for choosing a union branch. + + None means any shape -- a JSON value, or a union that mixes them. + """ + inner, _ = strip_annotation(annotation) + for registration in _CHECK_COMPILERS: + if registration.shape is not None and registration.predicate(inner): + return registration.shape + if inner is int: + return "int" + if inner is bool: + return "bool" + if inner is str: + return "str" + origin = get_origin(inner) + if origin is Literal: + values = get_args(inner) + return "int" if all(isinstance(value, int) for value in values) else "str" + if origin is tuple: + return "tuple" + if is_typeddict(inner) or is_dataclass(inner): + return "mapping" + return None + + +def has_shape(shape: str | None, value: object) -> bool: + if shape is None: + return True + if shape == "int": + return is_integer(value) + if shape == "bool": + return isinstance(value, bool) + if shape == "str": + return isinstance(value, str) + if shape == "tuple": + return isinstance(value, tuple) + if shape == "mapping": + return isinstance(value, Mapping) + return isinstance(value, (str, Mapping)) # "field" + + +def any_of(branches: Sequence[tuple[object, TypeCheck]], description: str) -> TypeCheck: + """A member whose type is a union of shapes, judged by the branch it fits. + + The branch whose top-level shape the value has is the one that + reports -- so an element inside a malformed array is located inside + the array, rather than the whole array being called wrong. A value + fitting no branch's shape is reported once, by what was expected. + """ + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + fitting = [ + check for annotation, check in branches if has_shape(shape_of(annotation), value) + ] + if len(fitting) == 0: + return problem(loc, f"expected {description}, got {value!r}") + verdicts = [check(value, loc) for check in fitting] + return () if any(len(verdict) == 0 for verdict in verdicts) else verdicts[0] + + return check + + +def fixed_tuple(elements: Sequence[TypeCheck], description: str) -> TypeCheck: + """A member whose type is an array of a fixed length, checked position by position.""" + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, tuple) or len(cast("tuple[object, ...]", value)) != len(elements): + return problem(loc, f"expected {description}, got {value!r}") + entries = cast("tuple[object, ...]", value) + return tuple( + found + for position, (element, entry) in enumerate(zip(elements, entries, strict=True)) + for found in element(entry, (*loc, position)) + ) + + return check + + +def mapping_of(members: Mapping[str, tuple[bool, TypeCheck]]) -> TypeCheck: + """A member that is itself an object with declared keys, checked key by key. + + Closed, like every configuration in this package: a key the type does + not declare is `unknown_key`, a required one missing is `missing_key`, + both located at the object. Each present member is checked at its own + key, so a problem inside is located inside. + """ + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(value, Mapping): + return problem(loc, f"expected an object, got {value!r}") + entries = cast("Mapping[str, object]", value) + found: list[ValidationProblem] = [] + for key in entries: + if key not in members: + found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) + for key, (required, member) in members.items(): + if key not in entries: + if required: + found.extend(problem(loc, f"missing required key {key!r}", "missing_key")) + continue + found.extend(member(entries[key], (*loc, key))) + return tuple(found) + + return check + + +def _members_of(annotations: Mapping[str, object]) -> dict[str, tuple[bool, TypeCheck]] | None: + """A member table for a nested object's keys; None if any key's type has no check.""" + members: dict[str, tuple[bool, TypeCheck]] = {} + for key, annotation in annotations.items(): + check = check_for(annotation) + if check is None: + return None + inner, _ = strip_annotation(annotation) + required = get_origin(annotation) is not NotRequired and not is_optional(inner) + members[key] = (required, check) + return members + + +CheckCompiler: TypeAlias = "Callable[[object], TypeCheck | None]" +"""Turns one annotation into its type check -- or None, to decline it after all.""" + + +@dataclass(frozen=True, slots=True) +class Registration: + """One shape `check_for` reads: how to recognise it, and what to make of it. + + `shape` and `description` are for a shape the compiler's own logic + does not know -- a nested metadata field, say -- so that choosing a + union branch and naming the shape in a message work for it too. + """ + + predicate: Callable[[object], bool] + compile: CheckCompiler + shape: str | None = None + description: str | None = None + + +_CHECK_COMPILERS: Final[list[Registration]] = [] +"""The shapes `check_for` reads, consulted front to back. + +The built-in shapes are appended below in the order they must be tried, +and `register_check` puts a registration in front of all of them, so the +newest one wins. `_entity` registers the nested metadata field this way, +ahead of the record shape -- an entity is a dataclass too. +""" + + +def register_check( + predicate: Callable[[object], bool], + compile: CheckCompiler, + *, + shape: str | None = None, + description: str | None = None, +) -> None: + """Teach `check_for` an annotation shape it does not read. + + Hex = NewType("Hex", str) + register_check(lambda annotation: annotation is Hex, lambda annotation: is_hex) + + `predicate` sees the annotation with `Annotated`, `NotRequired` and + `ReadOnly` peeled; `compile` returns the check for it, calling + `check_for` itself for any shape inside. A registration is consulted + before every built-in one, so a package can also replace how a + built-in shape is judged. The same door the built-ins came through, + which is what makes the set of shapes open rather than this module's. + + `shape` names the JSON shape the annotation admits, for choosing the + branch of a union a value fits (`"int"`, `"str"`, `"tuple"`, + `"mapping"`, or `"field"` for a bare name or object), and + `description` is how a message names it; both are needed only for a + shape the compiler's own logic does not recognise. + """ + _CHECK_COMPILERS.insert(0, Registration(predicate, compile, shape, description)) + + +def _builtin(predicate: Callable[[object], bool]) -> Callable[[CheckCompiler], CheckCompiler]: + """Register a built-in shape, in the order written.""" + + def append(compile: CheckCompiler) -> CheckCompiler: + _CHECK_COMPILERS.append(Registration(predicate, compile)) + return compile + + return append + + +@_builtin(lambda inner: inner is int) +def _compile_int(inner: object) -> TypeCheck | None: + return is_int + + +@_builtin(lambda inner: inner is bool) +def _compile_bool(inner: object) -> TypeCheck | None: + return is_bool + + +@_builtin(lambda inner: inner is str) +def _compile_str(inner: object) -> TypeCheck | None: + return is_str + + +@_builtin(lambda inner: inner is JSONValue) +def _compile_json_value(inner: object) -> TypeCheck | None: + return is_json_value + + +@_builtin(lambda inner: get_origin(inner) is Literal) +def _compile_literal(inner: object) -> TypeCheck | None: + # Sorted, because the order `get_args` reports is not the order the + # `Literal` was written in: two `Literal`s over the same values + # compare and hash equal, so the first one built anywhere in the + # process is the one every later one resolves to. The check is a + # membership test either way; this is so the message listing the + # values does not depend on import order. + return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) + + +@_builtin(is_union) +def _compile_union(inner: object) -> TypeCheck | None: + branches = [arg for arg in get_args(inner) if arg is not UNSET] + if len(branches) == 1: + return check_for(branches[0]) + compiled = [(branch, check_for(branch)) for branch in branches] + if any(check is None for _, check in compiled): + return None + return any_of( + [(branch, cast("TypeCheck", check)) for branch, check in compiled], describe(inner) + ) + + +@_builtin(lambda inner: get_origin(inner) is tuple) +def _compile_tuple(inner: object) -> TypeCheck | None: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = check_for(arguments[0]) + return None if element is None else sequence_of(element) + elements = [check_for(argument) for argument in arguments] + if any(element is None for element in elements): + return None + return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) + + +@_builtin(is_typeddict) +def _compile_typeddict(inner: object) -> TypeCheck | None: + members = _members_of(get_type_hints(inner, include_extras=True)) + return None if members is None else mapping_of(members) + + +@_builtin(lambda inner: isinstance(inner, type) and is_dataclass(inner)) +def _compile_record(inner: object) -> TypeCheck | None: + if not isinstance(inner, type): # pragma: no cover - the predicate says it is + return None + members = _members_of(field_hints(inner)) + return None if members is None else mapping_of(members) + + +def check_for(annotation: object) -> TypeCheck | None: + """The type check a field annotation implies, or None if it implies none. + + A small compiler over the shapes this package's metadata takes: the + JSON scalars, a `Literal` of names, arrays homogeneous or fixed, + unions of those, a nested object described by a TypedDict or a + record dataclass, and a nested metadata field -- an entity type, + with or without `Opaque`. `UNSET` in a union says the member may be + absent, which is the other half of a table entry and is read + separately by `is_optional`. + + Open: each shape is a registration in `_CHECK_COMPILERS`, and + `register_check` adds one from outside. None for an annotation no + registration claims, which the entity then declares a check for by + hand. + """ + inner, _ = strip_annotation(annotation) + for registration in _CHECK_COMPILERS: + if registration.predicate(inner): + return registration.compile(inner) + return None + + +def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], list[str]]: + """The member table an entity's own fields describe. + + Every field is a configuration member unless `FROM_NAME` says it is + carried by the envelope. Requiredness is whether the type admits + `UNSET`; the check is whatever `check_for` reads off the type. Also + returned: the fields no check could be read for, which the entity + must declare by hand. + """ + derived: dict[str, tuple[bool, TypeCheck]] = {} + unread: list[str] = [] + for name, annotation in field_hints(cls).items(): + inner, metadata = strip_annotation(annotation) + if any(entry is FROM_NAME for entry in metadata): + continue + check = check_for(inner) + if check is None: + unread.append(name) + continue + derived[name] = (not is_optional(inner), check) + return derived, unread + + +MemberRule: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" +"""A rule about one member: takes its value, reports relative to it.""" + + +_RULE_MEMBERS: Final[dict[object, tuple[str, ...]]] = {} +"""Which members each `@validates` rule is about, keyed by the function. + +A side table rather than an attribute on the function, so the decorator +hands back exactly what it was given -- the declared signature survives, +and the type checker keeps checking the body and its callers. +""" + + +_Rule = TypeVar("_Rule", bound="Callable[..., tuple[ValidationProblem, ...]]") + + +def rule_members(function: object) -> tuple[str, ...]: + """The members a function was marked as a rule about, if any.""" + return _RULE_MEMBERS.get(function, ()) + + +def validates(*members: str) -> Callable[[_Rule], _Rule]: + """Mark a static rule as being about one member, or several alike. + + @staticmethod + @validates("order") + def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: + ... + + The rule receives the member's value, already of the type the field + declares, and only when the member is present; it reports relative + to the member, so a problem with an empty location is about the + member itself. Naming several members applies the one rule to each. + A rule that reads two members together is `value_problems`. + """ + + def mark(rule: _Rule) -> _Rule: + _RULE_MEMBERS[rule] = members + return rule + + return mark + + +def _bound_check(metadata: Sequence[object]) -> TypeCheck | None: + """The check the bound markers among an annotation's metadata imply, or None.""" + ge = gt = le = lt = None + for marker in metadata: + if isinstance(marker, Ge): + ge = marker.bound + elif isinstance(marker, Gt): + gt = marker.bound + elif isinstance(marker, Le): + le = marker.bound + elif isinstance(marker, Lt): + lt = marker.bound + elif isinstance(marker, Interval): + ge = marker.ge if marker.ge is not None else ge + gt = marker.gt if marker.gt is not None else gt + le = marker.le if marker.le is not None else le + lt = marker.lt if marker.lt is not None else lt + if ge is None and gt is None and le is None and lt is None: + return None + if ge is not None and le is not None and gt is None and lt is None: + expectation = f"an integer in [{ge}, {le}]" + else: + comparisons = [ + text + for bound, text in ( + (ge, f">= {ge}"), + (gt, f"> {gt}"), + (le, f"<= {le}"), + (lt, f"< {lt}"), + ) + if bound is not None + ] + expectation = "an integer " + " and ".join(comparisons) + + def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + # Not a number: the type check's finding, not this one's. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return () + within_bounds = ( + (ge is None or value >= ge) + and (gt is None or value > gt) + and (le is None or value <= le) + and (lt is None or value < lt) + ) + if within_bounds: + return () + return problem(loc, f"expected {expectation}, got {value}", "invalid_value") + + return check + + +def value_check_for(annotation: object) -> TypeCheck | None: + """The value check an annotation's metadata implies, at any depth, or None. + + Over the shapes as the entity holds them, not as the JSON spells + them: this runs after every member has its type and every nested + entity has been read, so a record is a dataclass instance here and + an entity is skipped -- it is valid by construction. + """ + inner, metadata = strip_annotation(annotation) + own = _bound_check(metadata) + below: TypeCheck | None = None + if is_union(inner): + branches = [ + (branch, value_check_for(branch)) for branch in get_args(inner) if branch is not UNSET + ] + if any(check is not None for _, check in branches): + + def by_branch(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + for branch, check in branches: + if check is not None and has_shape(shape_of(branch), value): + return check(value, loc) + return () + + below = by_branch + elif get_origin(inner) is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = value_check_for(arguments[0]) + if element is not None: + each = element + + def per_element(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + entries = cast("tuple[object, ...]", value) + return tuple( + found + for position, entry in enumerate(entries) + for found in each(entry, (*loc, position)) + ) + + below = per_element + else: + positions = [value_check_for(argument) for argument in arguments] + if any(check is not None for check in positions): + + def per_position(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + entries = cast("tuple[object, ...]", value) + return tuple( + found + for position, (check, entry) in enumerate( + zip(positions, entries, strict=True) + ) + if check is not None + for found in check(entry, (*loc, position)) + ) + + below = per_position + elif isinstance(inner, type) and is_dataclass(inner) and shape_of(inner) != "field": + members = { + name: check + for name, field_annotation in field_hints(inner).items() + if (check := value_check_for(field_annotation)) is not None + } + if len(members) != 0: + + def per_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + return tuple( + found + for name, check in members.items() + if (held := getattr(value, name)) is not UNSET + for found in check(held, (*loc, name)) + ) + + below = per_field + if own is None and below is None: + return None + if below is None: + return own + if own is None: + return below + outer, inner_check = own, below + + def both(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + return (*outer(value, loc), *inner_check(value, loc)) + + return both + + +def element_annotations(inner: object, count: int) -> list[object]: + """The annotation of each element of a tuple type, one per element held.""" + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + return [arguments[0]] * count + return list(arguments) + + +def is_class_var(annotation: object) -> bool: + """Whether an annotation says `ClassVar`. + + `from __future__ import annotations` leaves them as strings, so this + reads the text when it gets one -- the same thing `dataclasses` does, + and for the same reason: resolving the name needs a module namespace + that is not available while the class is still being built. + """ + if isinstance(annotation, str): + stripped = annotation.strip() + return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) + return get_origin(annotation) is ClassVar + + +def declared_class_vars(cls: type) -> dict[str, type]: + """Every class variable annotated anywhere in `cls`'s ancestry. + + Mapped to the class that annotated it, so a message can say where the + requirement comes from. Base first, so a redeclaration names the + nearest ancestor. + """ + found: dict[str, type] = {} + for ancestor in reversed(cls.__mro__): + for name, annotation in own_annotations(ancestor).items(): + if is_class_var(annotation): + found[name] = ancestor + return found + + +__all__ = [ + "FROM_NAME", + "CheckCompiler", + "Ge", + "Gt", + "Interval", + "Le", + "Lt", + "MemberRule", + "any_of", + "check_for", + "declared_class_vars", + "derive_member_types", + "describe", + "element_annotations", + "field_hints", + "fixed_tuple", + "has_shape", + "is_class_var", + "is_optional", + "is_union", + "mapping_of", + "own_annotations", + "register_check", + "rule_members", + "shape_of", + "strip_annotation", + "validates", + "value_check_for", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 0c0cbb4fce..9c966c7bc9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -29,6 +29,7 @@ validate_array_metadata_v3 as validate_array_metadata_v3_structure, ) from zarr_metadata.v3._chain import chain_problems +from zarr_metadata.v3._compile import field_hints from zarr_metadata.v3._entity import ( CHUNK_GRID, CHUNK_KEY_ENCODING, @@ -43,7 +44,6 @@ Opaque, canonicalize_nested, contains_entity, - field_hints, render_nested, within, ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index a4deaf80a9..63c272f802 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -1,40 +1,27 @@ """What every metadata entity can do for itself. -A codec, data type, chunk grid or chunk key encoding is three things at -once: a JSON shape, a set of constraints on the values in that shape, and -a canonical spelling. Keeping the three apart put one entity's knowledge -in four modules and needed a table per axis plus a drift test per table to -hold them together. Here they are one class per entity, and the class is -where methods bind: - -- `coerce` is **type-space**: raw metadata in, the entity or the reasons - it is not that entity out. -- `problems` is **value-space**: the entity is well-typed by construction, - so this only asks whether its values are in range. -- `to_json` is **canonical**: the simplest spelling meaning the same, typed - as the entity's own object TypedDict. - -The TypedDicts stay: they model the JSON form, and the correspondence is -exact in both directions. A configuration TypedDict unpacked is the -dataclass constructor's signature, and the object TypedDict is what -`to_json` returns. `tests/v3/test_entities.py` asserts the first, so the -two cannot drift. - -Everything that needs the document or the codec chain stays outside, in -`zarr_metadata.rules`, because an entity cannot answer it alone. - -`coerce` takes a `Context`: the entities in scope for this reading. Most -entities ignore it -- a `gzip` codec is a `gzip` codec whatever else is -registered -- but the ones whose configuration contains other entities do -not. A `struct` data type holds field data types and a `sharding_indexed` -codec holds two codec pipelines, and neither can coerce its own -configuration without knowing what names are in scope inside it. - -The shared plumbing lives here too: the member checks every entity needs, -the compiler that reads them off a field annotation, and the walk over a -configuration that applies them. What stays with the entity is its fields --- the part that is about blosc rather than about entities -- and -everything the layer knows about a member is read from those. +A codec, data type, chunk grid or chunk key encoding is one frozen +dataclass whose fields are its schema. Everything the layer knows about +a member is read off the field annotations by `_compile`: which members +there are, which may be absent, how each is type-checked, the bounds it +must satisfy, and -- for a field typed as another entity -- that it is +read through the scope, written back as its own JSON, and put in +canonical form by recursing into it. The three places a value rule +lives, by what it is about: a bound on the field, a member's rule under +`@validates`, the members together in `value_problems`. + +`coerce` is the reading path: raw metadata in, the entity or the +reasons it is not one out, taking a `Context` -- the entities in scope +for this reading -- which the entities that contain other entities need. +`__init_subclass__` refuses, at class creation, every way of writing an +entity that would type-check and then misbehave somewhere that will not +name the class. + +Composition -- what needs the document or the codec chain -- is the +entity's to answer too, through `incoming_problems`, `shape_problems`, +`fill_value_problems`, `transition` and `grid`, each taking the part of +the document it needs. The document that composes those answers is +`_document`. """ from __future__ import annotations @@ -43,50 +30,85 @@ # (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class # creation, and a name that exists only for the type checker is a NameError # then -- for this package and for any tool introspecting an entity. -import sys -import types -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence # noqa: TC003 from copy import deepcopy from dataclasses import MISSING, Field, dataclass, fields, is_dataclass, replace from types import MappingProxyType from typing import ( TYPE_CHECKING, - Annotated, ClassVar, Final, Literal, - NotRequired, - Required, TypeAlias, TypeVar, - Union, cast, get_args, get_origin, get_type_hints, ) -from typing_extensions import ReadOnly, TypeIs, is_typeddict +from typing_extensions import is_typeddict -from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( MetadataValidationError, ValidationProblem, - is_json, ) from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: from typing import Self - from zarr_metadata.model._validation import ProblemKind from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._parts import ArrayParts from zarr_metadata.v3._registry import Context +from zarr_metadata.v3._checks import ( + Loc, + MemberTypes, + TypeCheck, + coerce_members, + is_bool, + is_int, + is_integer, + is_json_value, + is_metadata_field, + is_str, + named_configuration, + one_of, + problem, + sequence_of, + within, +) +from zarr_metadata.v3._compile import ( + FROM_NAME, + CheckCompiler, + Ge, + Gt, + Interval, + Le, + Lt, + MemberRule, + declared_class_vars, + derive_member_types, + element_annotations, + field_hints, + has_shape, + is_class_var, + is_optional, + is_union, + own_annotations, + register_check, + rule_members, + shape_of, + strip_annotation, + validates, + value_check_for, +) + EntityT = TypeVar("EntityT", bound="MetadataEntity") + # A real alias, not a string one: entity modules subscript it as # `Coerced[Self]` in a return annotation, and not all of them defer # annotation evaluation. @@ -102,7 +124,6 @@ problems to decide the verdict. They are different questions. """ -Loc: TypeAlias = "tuple[str | int, ...]" ExtensionPointField = Literal[ "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" @@ -115,16 +136,26 @@ which point it reads them at. """ + # Left to infer their `Literal` types rather than widened to # `ExtensionPointField`: `Context.coerce` overloads on the field, so a # call written with one of these constants gets the entity type back # rather than the base. They are still assignable to the alias. DATA_TYPE: Final = "data_type" + + CHUNK_GRID: Final = "chunk_grid" + + CHUNK_KEY_ENCODING: Final = "chunk_key_encoding" + + CODECS: Final = "codecs" + + STORAGE_TRANSFORMERS: Final = "storage_transformers" + StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How one scalar of a data type occupies bytes. @@ -133,6 +164,7 @@ member is about. """ + CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] """The three pipeline positions the v3 spec sorts codecs into. @@ -140,284 +172,6 @@ does not have a pipeline position, a codec does. """ -TypeCheck: TypeAlias = "Callable[[object, Loc], tuple[ValidationProblem, ...]]" -"""Whether one value has the type a member declares, and where if not.""" - -MemberTypes: TypeAlias = "Mapping[str, tuple[bool, TypeCheck]]" -"""Per configuration member: whether it is required, and its type check.""" - - -def problem( - loc: Loc, message: str, kind: ProblemKind = "invalid_type" -) -> tuple[ValidationProblem, ...]: - """One problem, as the tuple every check returns.""" - return (ValidationProblem(loc, message, kind),) - - -def is_integer(value: object) -> TypeIs[int]: - """A JSON integer: an `int`, and not a `bool`. - - `True` is an `int` in Python and `true` is not a number in JSON, so - the two have to be told apart everywhere a number is expected. - """ - return not isinstance(value, bool) and isinstance(value, int) - - -def is_int(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """An integer, and not a bool -- JSON `true` is not the integer 1.""" - if not is_integer(value): - return problem(loc, f"expected an integer, got {value!r}") - return () - - -def is_str(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, str): - return problem(loc, f"expected a string, got {value!r}") - return () - - -def is_bool(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, bool): - return problem(loc, f"expected a boolean, got {value!r}") - return () - - -def is_json_value(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """Any JSON value at all -- the widest type a member can declare.""" - if not is_json(value): - return problem(loc, f"expected a JSON value, got {value!r}") - return () - - -def one_of(allowed: tuple[str, ...]) -> TypeCheck: - """A member whose type is a closed set of names.""" - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if value not in allowed: - return problem(loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value") - return () - - return check - - -def sequence_of(element: TypeCheck) -> TypeCheck: - """A member whose type is a sequence, checked element by element.""" - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, (list, tuple)): - return problem(loc, f"expected a sequence, got {value!r}") - elements: tuple[object, ...] = tuple(cast("list[object] | tuple[object, ...]", value)) - return tuple( - found for index, entry in enumerate(elements) for found in element(entry, (*loc, index)) - ) - - return check - - -def _as_tuples(value: object) -> object: - """Every JSON array in `value`, at any depth, as a tuple. - - The TypedDicts spell a JSON array as a tuple throughout, so a member - taken straight from parsed JSON would otherwise hold a list where its - own type says tuple -- and two documents differing only in that would - compare unequal. - """ - if isinstance(value, (list, tuple)): - entries = cast("list[object] | tuple[object, ...]", value) - return tuple(_as_tuples(entry) for entry in entries) - if isinstance(value, Mapping): - entries = cast("Mapping[str, object]", value) - return {key: _as_tuples(entry) for key, entry in entries.items()} - return value - - -def coerce_members( - configuration: Mapping[str, object], types: MemberTypes -) -> tuple[dict[str, object], tuple[ValidationProblem, ...], frozenset[str]]: - """The members `types` declares, taken from `configuration`. - - Returns what was accepted, every problem found, and the names of the - required members that could not be read. Three kinds of problem, and - they differ in that last part: - - - a key the entity does not declare says the value carries something - extra, not that it is wrong; - - an *optional* member of the wrong type leaves that member absent, - and everything else about the entity is still readable -- a bad - `index_location` says nothing about whether a shard's pipelines - are well formed, and silencing them would lose a real judgment; - - a *required* member missing or of the wrong type does stop it. - There is no honest reading of a `blosc` whose level is a string. - """ - problems: list[ValidationProblem] = [] - members: dict[str, object] = {} - unreadable: set[str] = set() - for key in configuration: - if key not in types: - problems.extend( - problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") - ) - for key, (required, check) in types.items(): - if key not in configuration: - if required: - problems.extend( - problem(("configuration", key), f"missing required key {key!r}", "missing_key") - ) - unreadable.add(key) - continue - # Normalized before the check, so a check only ever sees the tuples - # the TypedDicts declare -- never the lists raw JSON arrives as. - value = _as_tuples(configuration[key]) - found = check(value, ("configuration", key)) - problems.extend(found) - # An unknown key says the value carries something extra, not that - # it is the wrong type -- so the member is still readable, and - # dropping it here would make `to_json` lose what was written. - if all(entry.kind == "unknown_key" for entry in found): - members[key] = value - elif required: - unreadable.add(key) - return members, tuple(problems), frozenset(unreadable) - - -class _FromName: - """The marker behind `FROM_NAME`.""" - - __slots__ = () - - def __repr__(self) -> str: - return "FROM_NAME" - - -FROM_NAME: Final = _FromName() -"""Marks a field carried by the metadata envelope's `name`, not its configuration. - - data_type_name: Annotated[str, FROM_NAME] - -A member all the same -- `value_problems` judges it -- but not a -configuration key, so it is neither read from nor written to a -`configuration` object. The raw-bytes family is the case: `r` keeps its -width in its name and has no configuration at all. -""" - - -@dataclass(frozen=True, slots=True) -class Ge: - """`Annotated[int, Ge(1)]`: the value is at least `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Gt: - """`Annotated[int, Gt(0)]`: the value is more than `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Le: - """`Annotated[int, Le(9)]`: the value is at most `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Lt: - """`Annotated[int, Lt(10)]`: the value is less than `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Interval: - """`Annotated[int, Interval(ge=0, le=9)]`: the value lies within these bounds. - - These five are the `annotated_types` vocabulary -- what pydantic reads - and msgspec's `Meta` mirrors -- so a reader recognises them. Defined - here rather than imported, so the package keeps its one dependency. - A bound is a value rule: it runs only once the member has the type it - declared, at whatever depth the annotation puts it, so a bound on an - array's element type judges each element at its own position. - """ - - ge: int | float | None = None - gt: int | float | None = None - le: int | float | None = None - lt: int | float | None = None - - -def _strip(annotation: object) -> tuple[object, tuple[object, ...]]: - """An annotation's type, and the metadata `Annotated` wrapped it in. - - `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict - key, not part of the value's type; peeled with the `Annotated` layers, - in whatever order they were written. - """ - metadata: list[object] = [] - while True: - origin = get_origin(annotation) - if origin is Annotated: - inner, *extras = get_args(annotation) - metadata.extend(extras) - annotation = inner - elif origin in (NotRequired, Required, ReadOnly): - (annotation,) = get_args(annotation) - else: - return annotation, tuple(metadata) - - -def _own_annotations(klass: type) -> dict[str, object]: - """A class's own annotations, unevaluated. - - From 3.14 a class does not carry an `__annotations__` dict until it is - asked for one, and asking evaluates every annotation at once -- so a - `ClassVar` naming something imported only for the type checker would - fail the whole class. `annotationlib` can hand them back as the text - they were written as, which is what the callers here want anyway: - class variables are skipped by text before anything is evaluated. - Earlier versions leave the dict on the class, strings or values as - the module chose. - """ - if sys.version_info >= (3, 14): - import annotationlib - - return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) - return dict(vars(klass).get("__annotations__", {})) - - -def field_hints(cls: type) -> dict[str, object]: - """The dataclass fields of `cls`, resolved, base first. - - Each class's own annotations are resolved in that class's module, - and class variables are skipped *before* resolving, by text -- so a - `ClassVar` whose annotation names something imported only for the - type checker cannot fail class creation. `@dataclass` sees the same - set, in the same order. - """ - hints: dict[str, object] = {} - for ancestor in reversed(cls.__mro__): - raw = { - name: annotation - for name, annotation in _own_annotations(ancestor).items() - if not _is_class_var(annotation) - } - if len(raw) == 0: - continue - shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) - hints.update(get_type_hints(shell, include_extras=True)) - return hints - - -def _is_union(annotation: object) -> bool: - return get_origin(annotation) in (Union, types.UnionType) - - -def is_optional(annotation: object) -> bool: - """Whether a field may be absent: its type admits `UNSET`.""" - inner, _ = _strip(annotation) - return _is_union(inner) and any(arg is UNSET for arg in get_args(inner)) - def _is_entity_type(candidate: object) -> bool: return candidate is Opaque or ( @@ -434,513 +188,32 @@ def _is_entity_or_opaque(candidates: Sequence[object]) -> bool: ) -def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """A nested metadata field: a bare name or a named-configuration object. - - Only the envelope's shape. Which entity the name denotes, and whether - its configuration is well formed, is settled when the containing - entity reads it in scope. - """ - if not isinstance(value, (str, Mapping)): - return problem(loc, f"expected a metadata field, got {value!r}") - return () - - -def describe(annotation: object) -> str: - """The annotation as a message would name it: "an integer", "an object".""" - inner, _ = _strip(annotation) - if inner is int: - return "an integer" - if inner is bool: - return "a boolean" - if inner is str: - return "a string" - if inner is JSONValue: - return "a JSON value" - origin = get_origin(inner) - if origin is Literal: - return f"one of {tuple(sorted(get_args(inner)))!r}" - if _is_union(inner): - branches = [arg for arg in get_args(inner) if arg is not UNSET] - if _is_entity_or_opaque(branches): - return "a metadata field" - return " or ".join(describe(branch) for branch in branches) - if origin is tuple: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - return f"an array of {describe(arguments[0])} elements" - if len(arguments) == 2: - return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" - return f"an array of {len(arguments)} elements" - if _is_entity_type(inner): - return "a metadata field" - if is_typeddict(inner) or is_dataclass(inner): - return "an object" - return "a value" - - -def _shape(annotation: object) -> str | None: - """The top-level JSON shape an annotation admits, for choosing a union branch. - - None means any shape -- a JSON value, or a union that mixes them. - """ - inner, _ = _strip(annotation) - if inner is int: - return "int" - if inner is bool: - return "bool" - if inner is str: - return "str" - origin = get_origin(inner) - if origin is Literal: - values = get_args(inner) - return "int" if all(isinstance(value, int) for value in values) else "str" - if origin is tuple: - return "tuple" - if _is_entity_type(inner): - return "field" - if is_typeddict(inner) or is_dataclass(inner): - return "mapping" - return None - - -def _has_shape(shape: str | None, value: object) -> bool: - if shape is None: - return True - if shape == "int": - return is_integer(value) - if shape == "bool": - return isinstance(value, bool) - if shape == "str": - return isinstance(value, str) - if shape == "tuple": - return isinstance(value, tuple) - if shape == "mapping": - return isinstance(value, Mapping) - return isinstance(value, (str, Mapping)) # "field" - - -def any_of(branches: Sequence[tuple[object, TypeCheck]], description: str) -> TypeCheck: - """A member whose type is a union of shapes, judged by the branch it fits. - - The branch whose top-level shape the value has is the one that - reports -- so an element inside a malformed array is located inside - the array, rather than the whole array being called wrong. A value - fitting no branch's shape is reported once, by what was expected. - """ - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - fitting = [check for annotation, check in branches if _has_shape(_shape(annotation), value)] - if len(fitting) == 0: - return problem(loc, f"expected {description}, got {value!r}") - verdicts = [check(value, loc) for check in fitting] - return () if any(len(verdict) == 0 for verdict in verdicts) else verdicts[0] - - return check - - -def fixed_tuple(elements: Sequence[TypeCheck], description: str) -> TypeCheck: - """A member whose type is an array of a fixed length, checked position by position.""" - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple) or len(cast("tuple[object, ...]", value)) != len(elements): - return problem(loc, f"expected {description}, got {value!r}") - entries = cast("tuple[object, ...]", value) - return tuple( - found - for position, (element, entry) in enumerate(zip(elements, entries, strict=True)) - for found in element(entry, (*loc, position)) - ) - - return check - - -def mapping_of(members: Mapping[str, tuple[bool, TypeCheck]]) -> TypeCheck: - """A member that is itself an object with declared keys, checked key by key. - - Closed, like every configuration in this package: a key the type does - not declare is `unknown_key`, a required one missing is `missing_key`, - both located at the object. Each present member is checked at its own - key, so a problem inside is located inside. - """ - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, Mapping): - return problem(loc, f"expected an object, got {value!r}") - entries = cast("Mapping[str, object]", value) - found: list[ValidationProblem] = [] - for key in entries: - if key not in members: - found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) - for key, (required, member) in members.items(): - if key not in entries: - if required: - found.extend(problem(loc, f"missing required key {key!r}", "missing_key")) - continue - found.extend(member(entries[key], (*loc, key))) - return tuple(found) - - return check - - -def _members_of(annotations: Mapping[str, object]) -> dict[str, tuple[bool, TypeCheck]] | None: - """A member table for a nested object's keys; None if any key's type has no check.""" - members: dict[str, tuple[bool, TypeCheck]] = {} - for key, annotation in annotations.items(): - check = check_for(annotation) - if check is None: - return None - inner, _ = _strip(annotation) - required = get_origin(annotation) is not NotRequired and not is_optional(inner) - members[key] = (required, check) - return members - - -CheckCompiler: TypeAlias = "Callable[[object], TypeCheck | None]" -"""Turns one annotation into its type check -- or None, to decline it after all.""" - -_CHECK_COMPILERS: Final[list[tuple[Callable[[object], bool], CheckCompiler]]] = [] -"""The shapes `check_for` reads, each as (does this annotation have it?, compile it). - -Consulted front to back. The built-in shapes are appended below in the -order they must be tried -- a nested metadata field before a record, -because `Opaque` is itself a dataclass -- and `register_check` puts a -registration in front of all of them, so the newest one wins. -""" - - -def register_check(predicate: Callable[[object], bool], compile: CheckCompiler) -> None: - """Teach `check_for` an annotation shape it does not read. - - Hex = NewType("Hex", str) - register_check(lambda annotation: annotation is Hex, lambda annotation: is_hex) - - `predicate` sees the annotation with `Annotated`, `NotRequired` and - `ReadOnly` peeled; `compile` returns the check for it, calling - `check_for` itself for any shape inside. A registration is consulted - before every built-in one, so a package can also replace how a - built-in shape is judged. The same door the built-ins came through, - which is what makes the set of shapes open rather than this module's. - """ - _CHECK_COMPILERS.insert(0, (predicate, compile)) - - -def _builtin(predicate: Callable[[object], bool]) -> Callable[[CheckCompiler], CheckCompiler]: - """Register a built-in shape, in the order written.""" - - def append(compile: CheckCompiler) -> CheckCompiler: - _CHECK_COMPILERS.append((predicate, compile)) - return compile - - return append - - -@_builtin(lambda inner: inner is int) -def _compile_int(inner: object) -> TypeCheck | None: - return is_int - - -@_builtin(lambda inner: inner is bool) -def _compile_bool(inner: object) -> TypeCheck | None: - return is_bool - - -@_builtin(lambda inner: inner is str) -def _compile_str(inner: object) -> TypeCheck | None: - return is_str - - -@_builtin(lambda inner: inner is JSONValue) -def _compile_json_value(inner: object) -> TypeCheck | None: - return is_json_value - - -@_builtin(lambda inner: get_origin(inner) is Literal) -def _compile_literal(inner: object) -> TypeCheck | None: - # Sorted, because the order `get_args` reports is not the order the - # `Literal` was written in: two `Literal`s over the same values - # compare and hash equal, so the first one built anywhere in the - # process is the one every later one resolves to. The check is a - # membership test either way; this is so the message listing the - # values does not depend on import order. - return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) - - -@_builtin(_is_union) -def _compile_union(inner: object) -> TypeCheck | None: - branches = [arg for arg in get_args(inner) if arg is not UNSET] - if len(branches) == 1: - return check_for(branches[0]) - if _is_entity_or_opaque(branches): - return is_metadata_field - compiled = [(branch, check_for(branch)) for branch in branches] - if any(check is None for _, check in compiled): - return None - return any_of( - [(branch, cast("TypeCheck", check)) for branch, check in compiled], describe(inner) - ) - - -@_builtin(lambda inner: get_origin(inner) is tuple) -def _compile_tuple(inner: object) -> TypeCheck | None: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - element = check_for(arguments[0]) - return None if element is None else sequence_of(element) - elements = [check_for(argument) for argument in arguments] - if any(element is None for element in elements): - return None - return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) +def _is_nested_field(annotation: object) -> bool: + """An entity type, or a union of entity types and `Opaque`.""" + candidates = list(get_args(annotation)) if is_union(annotation) else [annotation] + return _is_entity_or_opaque([candidate for candidate in candidates if candidate is not UNSET]) -# A nested metadata field, before the record shape: `Opaque` is itself a -# dataclass, and an entity type must not be walked as one either. -@_builtin(_is_entity_type) -def _compile_entity(inner: object) -> TypeCheck | None: +def _compile_nested_field(annotation: object) -> TypeCheck | None: return is_metadata_field -@_builtin(is_typeddict) -def _compile_typeddict(inner: object) -> TypeCheck | None: - members = _members_of(get_type_hints(inner, include_extras=True)) - return None if members is None else mapping_of(members) - - -@_builtin(lambda inner: isinstance(inner, type) and is_dataclass(inner)) -def _compile_record(inner: object) -> TypeCheck | None: - if not isinstance(inner, type): # pragma: no cover - the predicate says it is - return None - members = _members_of(field_hints(inner)) - return None if members is None else mapping_of(members) - - -def check_for(annotation: object) -> TypeCheck | None: - """The type check a field annotation implies, or None if it implies none. - - A small compiler over the shapes this package's metadata takes: the - JSON scalars, a `Literal` of names, arrays homogeneous or fixed, - unions of those, a nested object described by a TypedDict or a - record dataclass, and a nested metadata field -- an entity type, - with or without `Opaque`. `UNSET` in a union says the member may be - absent, which is the other half of a table entry and is read - separately by `is_optional`. - - Open: each shape is a registration in `_CHECK_COMPILERS`, and - `register_check` adds one from outside. None for an annotation no - registration claims, which the entity then declares a check for by - hand. - """ - inner, _ = _strip(annotation) - for predicate, compile in _CHECK_COMPILERS: - if predicate(inner): - return compile(inner) - return None - - -def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], list[str]]: - """The member table an entity's own fields describe. - - Every field is a configuration member unless `FROM_NAME` says it is - carried by the envelope. Requiredness is whether the type admits - `UNSET`; the check is whatever `check_for` reads off the type. Also - returned: the fields no check could be read for, which the entity - must declare by hand. - """ - derived: dict[str, tuple[bool, TypeCheck]] = {} - unread: list[str] = [] - for name, annotation in field_hints(cls).items(): - inner, metadata = _strip(annotation) - if any(entry is FROM_NAME for entry in metadata): - continue - check = check_for(inner) - if check is None: - unread.append(name) - continue - derived[name] = (not is_optional(inner), check) - return derived, unread - - -MemberRule: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" -"""A rule about one member: takes its value, reports relative to it.""" - -_RULE_MEMBERS: Final[dict[object, tuple[str, ...]]] = {} -"""Which members each `@validates` rule is about, keyed by the function. - -A side table rather than an attribute on the function, so the decorator -hands back exactly what it was given -- the declared signature survives, -and the type checker keeps checking the body and its callers. -""" - -_Rule = TypeVar("_Rule", bound="Callable[..., tuple[ValidationProblem, ...]]") - - -def validates(*members: str) -> Callable[[_Rule], _Rule]: - """Mark a static rule as being about one member, or several alike. - - @staticmethod - @validates("order") - def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: - ... - - The rule receives the member's value, already of the type the field - declares, and only when the member is present; it reports relative - to the member, so a problem with an empty location is about the - member itself. Naming several members applies the one rule to each. - A rule that reads two members together is `value_problems`. - """ - - def mark(rule: _Rule) -> _Rule: - _RULE_MEMBERS[rule] = members - return rule - - return mark - - -def _bound_check(metadata: Sequence[object]) -> TypeCheck | None: - """The check the bound markers among an annotation's metadata imply, or None.""" - ge = gt = le = lt = None - for marker in metadata: - if isinstance(marker, Ge): - ge = marker.bound - elif isinstance(marker, Gt): - gt = marker.bound - elif isinstance(marker, Le): - le = marker.bound - elif isinstance(marker, Lt): - lt = marker.bound - elif isinstance(marker, Interval): - ge = marker.ge if marker.ge is not None else ge - gt = marker.gt if marker.gt is not None else gt - le = marker.le if marker.le is not None else le - lt = marker.lt if marker.lt is not None else lt - if ge is None and gt is None and le is None and lt is None: - return None - if ge is not None and le is not None and gt is None and lt is None: - expectation = f"an integer in [{ge}, {le}]" - else: - comparisons = [ - text - for bound, text in ( - (ge, f">= {ge}"), - (gt, f"> {gt}"), - (le, f"<= {le}"), - (lt, f"< {lt}"), - ) - if bound is not None - ] - expectation = "an integer " + " and ".join(comparisons) - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - # Not a number: the type check's finding, not this one's. - if isinstance(value, bool) or not isinstance(value, (int, float)): - return () - within_bounds = ( - (ge is None or value >= ge) - and (gt is None or value > gt) - and (le is None or value <= le) - and (lt is None or value < lt) - ) - if within_bounds: - return () - return problem(loc, f"expected {expectation}, got {value}", "invalid_value") - - return check - - -def value_check_for(annotation: object) -> TypeCheck | None: - """The value check an annotation's metadata implies, at any depth, or None. - - Over the shapes as the entity holds them, not as the JSON spells - them: this runs after every member has its type and every nested - entity has been read, so a record is a dataclass instance here and - an entity is skipped -- it is valid by construction. - """ - inner, metadata = _strip(annotation) - own = _bound_check(metadata) - below: TypeCheck | None = None - if _is_union(inner): - branches = [ - (branch, value_check_for(branch)) for branch in get_args(inner) if branch is not UNSET - ] - if any(check is not None for _, check in branches): - - def by_branch(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - for branch, check in branches: - if check is not None and _has_shape(_shape(branch), value): - return check(value, loc) - return () - - below = by_branch - elif get_origin(inner) is tuple: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - element = value_check_for(arguments[0]) - if element is not None: - each = element - - def per_element(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - entries = cast("tuple[object, ...]", value) - return tuple( - found - for position, entry in enumerate(entries) - for found in each(entry, (*loc, position)) - ) - - below = per_element - else: - positions = [value_check_for(argument) for argument in arguments] - if any(check is not None for check in positions): - - def per_position(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - entries = cast("tuple[object, ...]", value) - return tuple( - found - for position, (check, entry) in enumerate( - zip(positions, entries, strict=True) - ) - if check is not None - for found in check(entry, (*loc, position)) - ) - - below = per_position - elif isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): - members = { - name: check - for name, field_annotation in field_hints(inner).items() - if (check := value_check_for(field_annotation)) is not None - } - if len(members) != 0: - - def per_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - return tuple( - found - for name, check in members.items() - if (held := getattr(value, name)) is not UNSET - for found in check(held, (*loc, name)) - ) - - below = per_field - if own is None and below is None: - return None - if below is None: - return own - if own is None: - return below - outer, inner_check = own, below - - def both(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - return (*outer(value, loc), *inner_check(value, loc)) - - return both +# The compiler knows nothing of entities; this is where it learns that a +# field typed as one is a nested metadata field. Registered ahead of every +# built-in shape -- an entity is a dataclass too, and must not be walked as +# a record -- through the same door a third party's shape comes in by. +register_check( + _is_nested_field, _compile_nested_field, shape="field", description="a metadata field" +) def contains_entity(annotation: object) -> bool: """Whether a value of this type holds a nested metadata field anywhere in it.""" - inner, _ = _strip(annotation) + inner, _ = strip_annotation(annotation) if _is_entity_type(inner): return True origin = get_origin(inner) - if _is_union(inner): + if is_union(inner): return any(contains_entity(arg) for arg in get_args(inner) if arg is not UNSET) if origin is tuple: return any(contains_entity(arg) for arg in get_args(inner) if arg is not Ellipsis) @@ -967,13 +240,13 @@ def _as_entity_kind(candidate: object) -> type[MetadataEntity] | None: def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: """Every entity type an annotation names, at any depth.""" - inner, _ = _strip(annotation) + inner, _ = strip_annotation(annotation) kind = _as_entity_kind(inner) if kind is not None: return [kind] origin = get_origin(inner) arguments: tuple[object, ...] = get_args(inner) - if _is_union(inner): + if is_union(inner): return [kind for arg in arguments if arg is not UNSET for kind in _entity_kinds(arg)] if origin is tuple: return [kind for arg in arguments if arg is not Ellipsis for kind in _entity_kinds(arg)] @@ -996,20 +269,12 @@ def _point_of(kind: type[MetadataEntity]) -> ExtensionPointField: return point -def _element_annotations(inner: object, count: int) -> list[object]: - """The annotation of each element of a tuple type, one per element held.""" - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - return [arguments[0]] * count - return list(arguments) - - def _fitting_branch(inner: object, value: object) -> object | None: """The branch of a union that holds an entity and whose shape `value` has.""" for branch in get_args(inner): if branch is UNSET or not contains_entity(branch): continue - if _has_shape(_shape(branch), value): + if has_shape(shape_of(branch), value): return branch return None @@ -1025,11 +290,11 @@ def _resolve( record holding one field by field. The value has passed its type check, so the shapes are the annotation's. """ - inner, _ = _strip(annotation) - candidates = list(get_args(inner)) if _is_union(inner) else [inner] + inner, _ = strip_annotation(annotation) + candidates = list(get_args(inner)) if is_union(inner) else [inner] if _is_entity_or_opaque(candidates): return context.coerce(_point_of(_entity_kinds(inner)[0]), value, loc) - if _is_union(inner): + if is_union(inner): branch = _fitting_branch(inner, value) return (value, ()) if branch is None else _resolve(branch, value, context, loc) if get_origin(inner) is tuple: @@ -1037,7 +302,7 @@ def _resolve( resolved: list[object] = [] found: list[ValidationProblem] = [] for position, (element, entry) in enumerate( - zip(_element_annotations(inner, len(entries)), entries, strict=True) + zip(element_annotations(inner, len(entries)), entries, strict=True) ): item, problems = _resolve(element, entry, context, (*loc, position)) resolved.append(item) @@ -1063,8 +328,8 @@ def render_nested(annotation: object, value: object) -> object: return value.to_json() if isinstance(value, Opaque): return value.json - inner, _ = _strip(annotation) - if _is_union(inner): + inner, _ = strip_annotation(annotation) + if is_union(inner): branch = _fitting_branch(inner, value) return value if branch is None else render_nested(branch, value) if get_origin(inner) is tuple: @@ -1072,7 +337,7 @@ def render_nested(annotation: object, value: object) -> object: return tuple( render_nested(element, entry) for element, entry in zip( - _element_annotations(inner, len(entries)), entries, strict=True + element_annotations(inner, len(entries)), entries, strict=True ) ) if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): @@ -1090,8 +355,8 @@ def canonicalize_nested(annotation: object, value: object) -> object: return value.canonical() if isinstance(value, Opaque): return value - inner, _ = _strip(annotation) - if _is_union(inner): + inner, _ = strip_annotation(annotation) + if is_union(inner): branch = _fitting_branch(inner, value) return value if branch is None else canonicalize_nested(branch, value) if get_origin(inner) is tuple: @@ -1099,7 +364,7 @@ def canonicalize_nested(annotation: object, value: object) -> object: return tuple( canonicalize_nested(element, entry) for element, entry in zip( - _element_annotations(inner, len(entries)), entries, strict=True + element_annotations(inner, len(entries)), entries, strict=True ) ) if ( @@ -1152,35 +417,6 @@ class Opaque: reason: Literal["out_of_scope", "invalid"] -def _is_class_var(annotation: object) -> bool: - """Whether an annotation says `ClassVar`. - - `from __future__ import annotations` leaves them as strings, so this - reads the text when it gets one -- the same thing `dataclasses` does, - and for the same reason: resolving the name needs a module namespace - that is not available while the class is still being built. - """ - if isinstance(annotation, str): - stripped = annotation.strip() - return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) - return get_origin(annotation) is ClassVar - - -def _declared_class_vars(cls: type) -> dict[str, type]: - """Every class variable annotated anywhere in `cls`'s ancestry. - - Mapped to the class that annotated it, so a message can say where the - requirement comes from. Base first, so a redeclaration names the - nearest ancestor. - """ - found: dict[str, type] = {} - for ancestor in reversed(cls.__mro__): - for name, annotation in _own_annotations(ancestor).items(): - if _is_class_var(annotation): - found[name] = ancestor - return found - - # No `slots=True`, deliberately. It rebuilds the class, which on Python # 3.11 and 3.12 leaves the zero-argument `super()` *in that same class's # body* pointing at the class it replaced. Several entities call `super()` @@ -1388,7 +624,7 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: # attribute is not even hashable. if not callable(function): continue - for member in _RULE_MEMBERS.get(function, ()): + for member in rule_members(function): if member not in hints: msg = f"{cls.__name__}: `@validates({member!r})` names no field of the entity" raise TypeError(msg) @@ -1408,11 +644,11 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: "or `ChunkGridEntity`" ) raise TypeError(msg) - annotated = _declared_class_vars(cls) + annotated = declared_class_vars(cls) shadowed = [ name - for name, annotation in _own_annotations(cls).items() - if name in annotated and annotated[name] is not cls and not _is_class_var(annotation) + for name, annotation in own_annotations(cls).items() + if name in annotated and annotated[name] is not cls and not is_class_var(annotation) ] if len(shadowed) != 0: # A field of that name would go into `member_types`, into the @@ -1829,50 +1065,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP return () -def within(prefix: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: - """One entity's problems, located in the document that holds it. - - An entity reports relative to its own `configuration`, so that is what - goes between the field and the member. A problem with an empty - location is about the entity itself -- a malformed `r` name, a - codec that cannot encode what reaches it -- and lands on the field. - """ - return tuple( - ValidationProblem( - (*prefix, *(("configuration", *found.loc) if len(found.loc) != 0 else ())), - found.message, - found.kind, - ) - for found in problems - ) - - -def named_configuration( - value: object, -) -> tuple[str | None, Mapping[str, object] | None, bool]: - """Split metadata into `(name, configuration, must_understand)`. - - The shared shape every entity arrives in: a bare name, or an object - carrying one. A `None` name means the value is not a metadata field at - all; a `None` configuration means the bare spelling was used. - """ - if isinstance(value, str): - return value, None, True - if not isinstance(value, Mapping): - return None, None, True - entry = cast("Mapping[str, object]", value) - name = entry.get("name") - if not isinstance(name, str): - return None, None, True - configuration = entry.get("configuration") - must_understand = entry.get("must_understand", True) - return ( - name, - cast("Mapping[str, object]", configuration) if isinstance(configuration, Mapping) else None, - must_understand if isinstance(must_understand, bool) else True, - ) - - __all__ = [ "CHUNK_GRID", "CHUNK_KEY_ENCODING", diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index f4b481bb26..eff7e20b9d 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -17,7 +17,7 @@ canonicalize_array_metadata_v3, validate_array_metadata_v3, ) -from zarr_metadata.v3._entity import _CHECK_COMPILERS +from zarr_metadata.v3._compile import _CHECK_COMPILERS from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.entity import ( @@ -620,5 +620,5 @@ class AcmeDigestCodec(CodecEntity): finally: # A registration is process-wide; leave the compiler as it was found. _CHECK_COMPILERS[:] = [ - entry for entry in _CHECK_COMPILERS if entry[0] is not is_hex_annotation + entry for entry in _CHECK_COMPILERS if entry.predicate is not is_hex_annotation ] From a055e6c1d7e5ac538765e3a42590089112398c6a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 11:31:34 +0200 Subject: [PATCH 067/107] build(zarr-metadata): just test-versions runs the oldest and newest interpreters `just check` was described as everything CI runs for this package, and it ran the tests on one interpreter. CI runs them on four, and the 3.14 job is the one that caught `vars(cls)["__annotations__"]` coming back empty under PEP 649 -- a difference no run on 3.11 could see. `test-versions` runs the suite on the floor and the ceiling of the declared range, reading both from pyproject.toml -- `requires-python` and the newest classifier -- so the recipe cannot drift from what CI's matrix spans. `check` runs it in place of `test`; `test` stays for the quick single-interpreter run. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/justfile | 34 ++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index d24007983f..ef55a98f38 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -8,10 +8,38 @@ set positional-arguments default: @just --list -# Run the test suite; extra args are passed to pytest +# Run the test suite on the project interpreter; extra args are passed to pytest test *args: uv run --group test pytest tests "$@" +# The oldest and newest interpreters the package is declared for: the floor is +# `requires-python`, the ceiling the newest classifier, both read from +# pyproject.toml so this recipe cannot drift from what CI's matrix spans. +# uv fetches an interpreter it does not have. A difference in how a version +# materialises annotations or resolves a stub shows up here, not only in CI. +# Run the test suite on the oldest and newest supported interpreters +test-versions *args: + #!/usr/bin/env bash + set -euo pipefail + read -r floor ceiling < <(uv run python - <<'EOF' + import re + import tomllib + + project = tomllib.load(open("pyproject.toml", "rb"))["project"] + floor = re.fullmatch(r">=\s*(\d+\.\d+)", project["requires-python"]).group(1) + declared = [ + classifier.rsplit(" ", 1)[1] + for classifier in project["classifiers"] + if classifier.startswith("Programming Language :: Python :: 3.") + ] + print(floor, max(declared, key=lambda version: tuple(map(int, version.split("."))))) + EOF + ) + for version in "$floor" "$ceiling"; do + echo "== python $version ==" + uv run --python "$version" --group test pytest tests "$@" + done + # Lint with the same invocation CI uses lint: uvx ruff check . @@ -26,8 +54,8 @@ pyright_version := "1.1.414" typecheck: uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright -# Run everything CI runs for this package -check: lint typecheck test docs-check +# Run everything CI runs for this package, on both ends of the version range +check: lint typecheck test-versions docs-check # Preview the changelog that the next release would generate changelog-draft: From 79e65c4f49b7d73fe109cd0de098cfe3c6c21e4a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 12:06:56 +0200 Subject: [PATCH 068/107] refactor(zarr-metadata): to_json is typed by the entity's own JSON type An entity names its JSON type as its base's argument -- `class GzipCodec(CodecEntity[GzipCodecMetadata])` -- and `to_json` returns it. The parameter is covariant and defaults to the metadata field type, so a bare `CodecEntity` still admits every codec and a third party may leave it unnamed. The 17 overrides that narrowed the base's result by cast are gone; the one cast left is the seam in the base, and it is held to twice: class creation refuses a named type whose shape disagrees with what the members make the entity write, and the tests run every entity's output through the compiled named type. `crc32c` names `Crc32cCodecName` alone, since it never writes an object; the union is what a document may spell. `check_for` reads `Mapping[str, V]` and `NewType` now, so the public envelope TypedDicts compile; `is_entity` narrows an `object` to the defaulted base rather than `MetadataEntity[Unknown]`; `_chain` takes `Sequence[CodecEntity | Opaque]`, which every caller passes. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 2 +- packages/zarr-metadata/changes/4379.misc.2.md | 37 ++- .../src/zarr_metadata/rules/_documents.py | 3 +- .../src/zarr_metadata/v3/_chain.py | 10 +- .../src/zarr_metadata/v3/_checks.py | 16 ++ .../src/zarr_metadata/v3/_compile.py | 38 ++- .../src/zarr_metadata/v3/_entity.py | 130 +++++++-- .../v3/chunk_grid/rectilinear.py | 5 +- .../zarr_metadata/v3/chunk_grid/regular.py | 5 +- .../v3/chunk_key_encoding/default.py | 9 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 7 +- .../src/zarr_metadata/v3/codec/blosc.py | 7 +- .../src/zarr_metadata/v3/codec/bytes.py | 7 +- .../src/zarr_metadata/v3/codec/cast_value.py | 7 +- .../src/zarr_metadata/v3/codec/crc32c.py | 7 +- .../src/zarr_metadata/v3/codec/gzip.py | 7 +- .../zarr_metadata/v3/codec/scale_offset.py | 7 +- .../v3/codec/sharding_indexed.py | 7 +- .../src/zarr_metadata/v3/codec/transpose.py | 7 +- .../src/zarr_metadata/v3/codec/zstd.py | 7 +- .../zarr_metadata/v3/data_type/_families.py | 9 +- .../src/zarr_metadata/v3/data_type/bool.py | 2 +- .../src/zarr_metadata/v3/data_type/bytes.py | 2 +- .../zarr_metadata/v3/data_type/complex128.py | 2 +- .../zarr_metadata/v3/data_type/complex64.py | 2 +- .../src/zarr_metadata/v3/data_type/float16.py | 2 +- .../src/zarr_metadata/v3/data_type/float32.py | 2 +- .../src/zarr_metadata/v3/data_type/float64.py | 2 +- .../src/zarr_metadata/v3/data_type/int16.py | 2 +- .../src/zarr_metadata/v3/data_type/int32.py | 2 +- .../src/zarr_metadata/v3/data_type/int64.py | 2 +- .../src/zarr_metadata/v3/data_type/int8.py | 2 +- .../v3/data_type/numpy_datetime64.py | 7 +- .../v3/data_type/numpy_timedelta64.py | 7 +- .../src/zarr_metadata/v3/data_type/raw.py | 9 +- .../src/zarr_metadata/v3/data_type/string.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 5 +- .../src/zarr_metadata/v3/data_type/uint16.py | 2 +- .../src/zarr_metadata/v3/data_type/uint32.py | 2 +- .../src/zarr_metadata/v3/data_type/uint64.py | 2 +- .../src/zarr_metadata/v3/data_type/uint8.py | 2 +- .../src/zarr_metadata/v3/entity.py | 4 +- .../zarr-metadata/tests/v3/test_entities.py | 257 ++++++++++++++---- .../tests/v3/test_extension_api.py | 62 ++++- 44 files changed, 508 insertions(+), 209 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index a1bae27f8b..5e04a3d35d 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -96,7 +96,7 @@ The set of annotation shapes the compiler reads is open. Each built-in shape is a registration -- a predicate over the annotation and what to compile it to -- consulted in order, and `register_check` adds one from outside, ahead of the built-ins, so a package with a field type this -package does not read (a `NewType`, say) teaches the compiler once +package does not read (a `str` subclass, say) teaches the compiler once rather than declaring a check on every entity that uses it. The same door the built-in shapes came through; cattrs' `register_structure_hook_func` is the pattern. diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 564c6da470..7929b39123 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -21,17 +21,28 @@ against the rule registry this layer replaced, over one shared corpus of 40,000 documents, no verdict differs in the laxer direction and the valid documents report identically. -One thing deliberately not done. An entity's `to_json` returns its own -object TypedDict, which mypy will not accept where a -`ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as -`Mapping[str, object]`, never as the `Mapping[str, JSONValue]` the -envelope declares, so the write path needs a `cast` under mypy. +An entity names its own JSON type as the base's argument -- +`class GzipCodec(CodecEntity[GzipCodecMetadata])` -- and `to_json` returns +it. That is the one place the correspondence between an entity and its +public JSON type is written, and it is held to twice: class creation +refuses a named type whose shape (bare name, object, or either) disagrees +with what the members make the entity write, and a property test draws +documents from the named type, reads them in, writes them back, and judges +the result against the named type with the package's own compiler. The +parameter has a default, so a bare `CodecEntity` -- in a field, a table, a +scope -- admits every codec, and a third party may leave it unnamed. +`crc32c` names `Crc32cCodecName` alone: it has no members, so it only +ever writes the bare name, and the union is what a document may spell, +not what the entity writes. For the conformance test to use the +package's own compiler as its oracle, `check_for` now reads two shapes +it did not: `Mapping[str, V]`, an object of undeclared keys, and a +`NewType`, as the type it names -- which also closes a gap for a +third-party field typed with either. -The conversion is sound and the annotation stays. Mypy's rule exists -because an ordinary TypedDict may carry extra items of undeclared types, -so the union of the declared value types does not bound the mapping; -every TypedDict here is `closed` (PEP 728), which forbids that, and -pyright implements PEP 728 while mypy does not yet (python/mypy#8994, -python/mypy#18439). Widening `configuration` would satisfy mypy by making -the annotation say something false, and a configuration's values are -JSON. `zarr_metadata.v3.entity` records the `cast` and the reason. +One thing this does not change, under mypy. An entity's JSON type is a +TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is +wanted: it reads every TypedDict as `Mapping[str, object]`, never as the +`Mapping[str, JSONValue]` the envelope declares (python/mypy#8994, +python/mypy#18439 -- mypy lacks PEP 728). The conversion is sound and the +annotation stays; a consumer under mypy casts at the one place it puts an +entity's JSON into a document. diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index e106091075..5695209a51 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -251,8 +251,7 @@ def canonicalize_array_metadata_v3( problems = (*problems, *found, *array.problems()) if len(problems) != 0: return Invalid(problems) - canonical = cast("ZarrV3ArrayMetadataJSON", array.canonical().to_json()) - return Canonical(ZarrV3ArrayMetadata.from_json(canonical).to_json()) + return Canonical(ZarrV3ArrayMetadata.from_json(array.canonical().to_json()).to_json()) __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py index 08d659dfaa..411b5c0e80 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py @@ -28,19 +28,21 @@ if TYPE_CHECKING: from collections.abc import Sequence - from zarr_metadata.v3._entity import Loc + from zarr_metadata.v3._entity import Loc, Opaque from zarr_metadata.v3._parts import ArrayParts _KIND_RANK = {"array_array": 0, "array_bytes": 1, "bytes_bytes": 2} -def _label(codec: object) -> str: +def _label(codec: CodecEntity | Opaque) -> str: if isinstance(codec, CodecEntity): return repr(type(codec).identifier) return repr(codec) -def order_problems(codecs: Sequence[object], loc: Loc) -> tuple[ValidationProblem, ...]: +def order_problems( + codecs: Sequence[CodecEntity | Opaque], loc: Loc +) -> tuple[ValidationProblem, ...]: """Whether the pipeline is shaped the way the spec orders it. A codec out of scope is skipped: it imposes no ordering constraint, @@ -84,7 +86,7 @@ def order_problems(codecs: Sequence[object], loc: Loc) -> tuple[ValidationProble def chain_problems( - codecs: Sequence[object], start: ArrayParts | None, loc: Loc + codecs: Sequence[CodecEntity | Opaque], start: ArrayParts | None, loc: Loc ) -> tuple[ValidationProblem, ...]: """Every problem this pipeline has, ordering and per-codec alike. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py index a2ee3cc233..477611515b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py @@ -112,6 +112,22 @@ def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return check +def object_of(value: TypeCheck) -> TypeCheck: + """A member whose type is an object with any keys, checked value by value. + + The open counterpart of `mapping_of`: a `Mapping[str, V]` says nothing + about which keys there are, only what each value must be. + """ + + def check(candidate: object, loc: Loc) -> tuple[ValidationProblem, ...]: + if not isinstance(candidate, Mapping): + return problem(loc, f"expected an object, got {candidate!r}") + entries = cast("Mapping[str, object]", candidate) + return tuple(found for key, entry in entries.items() for found in value(entry, (*loc, key))) + + return check + + def _as_tuples(value: object) -> object: """Every JSON array in `value`, at any depth, as a tuple. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py index 5f720cc91d..12f90ef911 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py @@ -32,6 +32,7 @@ ClassVar, Final, Literal, + NewType, NotRequired, Required, TypeAlias, @@ -53,6 +54,7 @@ is_integer, is_json_value, is_str, + object_of, one_of, problem, sequence_of, @@ -229,6 +231,10 @@ def describe(annotation: object) -> str: if len(arguments) == 2: return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" return f"an array of {len(arguments)} elements" + if origin in (Mapping, dict): + return "an object" + if isinstance(inner, NewType): + return describe(inner.__supertype__) if is_typeddict(inner) or is_dataclass(inner): return "an object" return "a value" @@ -255,6 +261,10 @@ def shape_of(annotation: object) -> str | None: return "int" if all(isinstance(value, int) for value in values) else "str" if origin is tuple: return "tuple" + if origin in (Mapping, dict): + return "mapping" + if isinstance(inner, NewType): + return shape_of(inner.__supertype__) if is_typeddict(inner) or is_dataclass(inner): return "mapping" return None @@ -392,7 +402,7 @@ def register_check( ) -> None: """Teach `check_for` an annotation shape it does not read. - Hex = NewType("Hex", str) + class Hex(str): ... register_check(lambda annotation: annotation is Hex, lambda annotation: is_hex) `predicate` sees the annotation with `Annotated`, `NotRequired` and @@ -491,16 +501,34 @@ def _compile_record(inner: object) -> TypeCheck | None: return None if members is None else mapping_of(members) +@_builtin(lambda inner: get_origin(inner) in (Mapping, dict)) +def _compile_mapping(inner: object) -> TypeCheck | None: + # An object of undeclared keys: `Mapping[str, V]`, every value a `V`. + arguments = get_args(inner) + if len(arguments) != 2 or arguments[0] is not str: + return None + value = check_for(arguments[1]) + return None if value is None else object_of(value) + + +@_builtin(lambda inner: isinstance(inner, NewType)) +def _compile_new_type(inner: object) -> TypeCheck | None: + # A `NewType` is its supertype to a document; the distinction is the + # code's, for a value it has vouched for. + return check_for(cast("NewType", inner).__supertype__) + + def check_for(annotation: object) -> TypeCheck | None: """The type check a field annotation implies, or None if it implies none. A small compiler over the shapes this package's metadata takes: the JSON scalars, a `Literal` of names, arrays homogeneous or fixed, unions of those, a nested object described by a TypedDict or a - record dataclass, and a nested metadata field -- an entity type, - with or without `Opaque`. `UNSET` in a union says the member may be - absent, which is the other half of a table entry and is read - separately by `is_optional`. + record dataclass, an object of undeclared keys as `Mapping[str, V]`, + a `NewType` as the type it names, and a nested metadata field -- an + entity type, with or without `Opaque`. `UNSET` in a union says the + member may be absent, which is the other half of a table entry and + is read separately by `is_optional`. Open: each shape is a registration in `_CHECK_COMPILERS`, and `register_check` adds one from outside. None for an annotation no diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 63c272f802..cc0abf14e9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -38,16 +38,16 @@ TYPE_CHECKING, ClassVar, Final, + Generic, Literal, TypeAlias, - TypeVar, cast, get_args, get_origin, get_type_hints, ) -from typing_extensions import is_typeddict +from typing_extensions import TypeIs, TypeVar, is_typeddict from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( @@ -59,7 +59,6 @@ if TYPE_CHECKING: from typing import Self - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._parts import ArrayParts from zarr_metadata.v3._registry import Context @@ -80,6 +79,7 @@ sequence_of, within, ) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._compile import ( FROM_NAME, CheckCompiler, @@ -108,6 +108,21 @@ EntityT = TypeVar("EntityT", bound="MetadataEntity") +JSONT_co = TypeVar( + "JSONT_co", bound=ZarrV3MetadataFieldJSON, default=ZarrV3MetadataFieldJSON, covariant=True +) +"""What an entity's `to_json` returns: its own JSON type, named as the base's argument. + + class GzipCodec(CodecEntity[GzipCodecMetadata]): ... + +Covariant, because it appears only in a return; defaulted, so a bare +`CodecEntity` -- in a field annotation, a table of entities, a scope -- +means `CodecEntity[ZarrV3MetadataFieldJSON]` and admits every codec, each +of whose JSON types is assignable to that one (`ZarrV3NamedConfigJSON` is +`ReadOnly` and closed for exactly this). An entity that leaves it +defaulted is not wrong, only less informative. +""" + # A real alias, not a string one: entity modules subscript it as # `Coerced[Self]` in a return annotation, and not all of them defer @@ -173,10 +188,61 @@ """ +def is_entity(value: object) -> TypeIs[MetadataEntity]: + """`value` is an entity, of whatever JSON type. + + An `isinstance` against the generic base narrows an `object` to + `MetadataEntity[Unknown]`; this narrows it to the defaulted + `MetadataEntity`, whose `to_json` is any metadata field -- which is + all that can be said of an entity met as an `object`. + """ + return isinstance(value, MetadataEntity) + + +def _is_entity_kind(candidate: object) -> TypeIs[type[MetadataEntity]]: + """`candidate` is an entity class; narrowed as `is_entity` narrows.""" + return isinstance(candidate, type) and issubclass(candidate, MetadataEntity) + + +def _unsubscripted(candidate: object) -> object: + """`CodecEntity[X]` as `CodecEntity`; anything else as it is.""" + origin = get_origin(candidate) + return origin if isinstance(origin, type) else candidate + + def _is_entity_type(candidate: object) -> bool: - return candidate is Opaque or ( - isinstance(candidate, type) and issubclass(candidate, MetadataEntity) + candidate = _unsubscripted(candidate) + return candidate is Opaque or _is_entity_kind(candidate) + + +def json_type_of(cls: type[MetadataEntity]) -> object: + """The JSON type `cls` names for `to_json`; the default if it names none. + + Read off the subscripted base the class -- or the nearest ancestor + that did -- was declared with, `CodecEntity[GzipCodecMetadata]`: the + one place the correspondence between an entity and its public JSON + type is written. + """ + for klass in cls.__mro__: + for base in klass.__dict__.get("__orig_bases__", ()): + origin = get_origin(base) + if isinstance(origin, type) and issubclass(origin, MetadataEntity): + arguments = get_args(base) + if len(arguments) == 1 and not isinstance(arguments[0], TypeVar): + return arguments[0] + return ZarrV3MetadataFieldJSON + + +def _json_shape(json_type: object) -> tuple[bool, bool]: + """Whether a JSON type admits a bare name, and whether it admits an object.""" + parts = get_args(json_type) if is_union(json_type) else (json_type,) + bare = any( + part is str + or (get_origin(part) is Literal and all(isinstance(v, str) for v in get_args(part))) + or getattr(part, "__supertype__", None) is str + for part in parts ) + return bare, any(is_typeddict(part) for part in parts) def _is_entity_or_opaque(candidates: Sequence[object]) -> bool: @@ -233,7 +299,8 @@ def _as_entity_kind(candidate: object) -> type[MetadataEntity] | None: narrows this parameter and not the caller's variable, which the caller goes on to read as the annotation it is. """ - if isinstance(candidate, type) and issubclass(candidate, MetadataEntity): + candidate = _unsubscripted(candidate) + if _is_entity_kind(candidate): return candidate return None @@ -324,7 +391,7 @@ def _resolve( def render_nested(annotation: object, value: object) -> object: """`value` as a document would write it: every nested entity in its JSON form.""" - if isinstance(value, MetadataEntity): + if is_entity(value): return value.to_json() if isinstance(value, Opaque): return value.json @@ -351,7 +418,7 @@ def render_nested(annotation: object, value: object) -> object: def canonicalize_nested(annotation: object, value: object) -> object: """`value` with every nested entity in its own canonical form.""" - if isinstance(value, MetadataEntity): + if is_entity(value): return value.canonical() if isinstance(value, Opaque): return value @@ -425,7 +492,7 @@ class Opaque: # that is the floor this is worth revisiting; the memory saved is small at # document scale, which is why it has not been. @dataclass(frozen=True) -class MetadataEntity: +class MetadataEntity(Generic[JSONT_co]): """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what @@ -605,6 +672,24 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: raise TypeError(msg) cls.member_types = {**derived, **declared} cls.configuration_required = any(required for required, _ in cls.member_types.values()) + json_type = json_type_of(cls) + if json_type is not ZarrV3MetadataFieldJSON: + # The named type is a promise about what `to_json` writes, and + # its shape follows from the members: a bare name only when no + # member is required and the entity must be understood, an + # object whenever there is a member to write or the flag to. + admits_bare, admits_object = _json_shape(json_type) + writes_bare = not cls.configuration_required and cls.must_understand + writes_object = len(cls.member_types) != 0 or not cls.must_understand + if admits_bare != writes_bare or admits_object != writes_object: + msg = ( + f"{cls.__name__} names {json_type!r} as its JSON type, which " + f"{'admits' if admits_bare else 'lacks'} a bare name and " + f"{'admits' if admits_object else 'lacks'} an object, but the entity " + f"{'writes' if writes_bare else 'never writes'} a bare name and " + f"{'writes' if writes_object else 'never writes'} an object" + ) + raise TypeError(msg) hints = field_hints(cls) cls.nested_members = { name: annotation for name, annotation in hints.items() if contains_entity(annotation) @@ -933,7 +1018,7 @@ def unchecked(cls, **members: object) -> Self: raise TypeError(msg) return entity - def to_json(self) -> ZarrV3MetadataFieldJSON: + def to_json(self) -> JSONT_co: """This entity as a document would write it. Faithful to every member it models: read a document, write it @@ -954,22 +1039,30 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: `must_understand` follows the entity's own class variable, so it is omitted for everything this package models today. - Subclasses narrow the return type to their own object TypedDict, - which is the JSON form this dataclass models. + The return type is the entity's own JSON type, named as the + base's argument -- `GzipCodecObject`, `BytesCodecObject | + BytesCodecName` -- and the `cast` below is the one place the + package asserts that the dict it builds has that shape. Asserted + rather than proven because a TypedDict cannot be built member by + member from `dict[str, object]`; held to, twice: `__init_subclass__` + refuses a named type whose shape disagrees with whether this + entity ever writes a bare name or an object, and + `tests/v3/test_entities.py` compiles the named type with + `check_for` and runs every entity's output through it. """ configuration = self.configuration() if len(configuration) == 0 and type(self).must_understand: - return cast("ZarrV3MetadataFieldJSON", type(self).identifier) + return cast("JSONT_co", type(self).identifier) entry: dict[str, object] = {"name": type(self).identifier} if len(configuration) != 0: entry["configuration"] = configuration if not type(self).must_understand: entry["must_understand"] = False - return cast("ZarrV3MetadataFieldJSON", entry) + return cast("JSONT_co", entry) @dataclass(frozen=True) -class CodecEntity(MetadataEntity, base=True): +class CodecEntity(MetadataEntity[JSONT_co], base=True): """An entity that occupies a position in the codec pipeline.""" extension_point: ClassVar[ExtensionPointField] = CODECS @@ -1010,7 +1103,7 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: @dataclass(frozen=True) -class ChunkGridEntity(MetadataEntity, base=True): +class ChunkGridEntity(MetadataEntity[JSONT_co], base=True): """An entity that divides an array into the parts a pipeline encodes.""" extension_point: ClassVar[ExtensionPointField] = CHUNK_GRID @@ -1034,7 +1127,7 @@ def grid(self, array_shape: object) -> ChunkGrid: @dataclass(frozen=True) -class DataTypeEntity(MetadataEntity, base=True): +class DataTypeEntity(MetadataEntity[JSONT_co], base=True): """An entity that says how the array's scalars are stored. Only data types answer that, and every rule that turns on it -- a @@ -1082,6 +1175,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "Ge", "Gt", "Interval", + "JSONT_co", "Le", "Loc", "Lt", @@ -1094,11 +1188,13 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "ValueRoutine", "coerce_members", "is_bool", + "is_entity", "is_int", "is_integer", "is_json_value", "is_metadata_field", "is_str", + "json_type_of", "named_configuration", "one_of", "problem", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 46115ece82..f86ac9268f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -172,7 +172,7 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: @dataclass(frozen=True) -class RectilinearChunkGrid(ChunkGridEntity): +class RectilinearChunkGrid(ChunkGridEntity[RectilinearChunkGridMetadata]): """The `rectilinear` chunk grid, coerced from its metadata.""" kind: Literal["inline"] @@ -228,6 +228,3 @@ def canonical(self) -> Self: grid, and the encoded one stays the same size as the array grows. """ return replace(super().canonical(), chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) - - def to_json(self) -> RectilinearChunkGridObject: - return cast("RectilinearChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 63f08ade58..e10a8f3cb1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -62,7 +62,7 @@ class RegularChunkGridObject(TypedDict, closed=True): @dataclass(frozen=True) -class RegularChunkGrid(ChunkGridEntity): +class RegularChunkGrid(ChunkGridEntity[RegularChunkGridMetadata]): """The `regular` chunk grid, coerced from its metadata.""" chunk_shape: tuple[Annotated[int, Ge(1)], ...] @@ -86,6 +86,3 @@ def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: def grid(self, array_shape: object) -> ChunkGrid: """One extent per axis, the same for every chunk on that axis.""" return ChunkGrid.regular(self.chunk_shape) - - def to_json(self) -> RegularChunkGridObject: - return cast("RegularChunkGridObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index a22b52383f..0199fc61a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -8,7 +8,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -71,14 +71,9 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class DefaultChunkKeyEncoding(MetadataEntity): +class DefaultChunkKeyEncoding(MetadataEntity[DefaultChunkKeyEncodingMetadata]): """The `default` chunk key encoding, coerced from its metadata.""" separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME - - def to_json(self) -> DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName: - return cast( - "DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName", super().to_json() - ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 2abc10f872..57e5ed0749 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -14,7 +14,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -77,12 +77,9 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class V2ChunkKeyEncoding(MetadataEntity): +class V2ChunkKeyEncoding(MetadataEntity[V2ChunkKeyEncodingMetadata]): """The `v2` chunk key encoding, coerced from its metadata.""" separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME - - def to_json(self) -> V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName: - return cast("V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 28cf6b28a9..f29685c262 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass, replace -from typing import Annotated, ClassVar, Final, Literal, NotRequired, Self, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict, Unpack @@ -91,7 +91,7 @@ class BloscCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BloscCodec(CodecEntity): +class BloscCodec(CodecEntity[BloscCodecMetadata]): """The `blosc` codec, coerced from its metadata. Everything blosc knows about itself: the shape its metadata takes, the @@ -152,6 +152,3 @@ def canonical(self) -> Self: if canonical.shuffle != BLOSC_NO_SHUFFLE or canonical.typesize is UNSET: return canonical return replace(canonical, typesize=UNSET) - - def to_json(self) -> BloscCodecObject: - return cast("BloscCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 8016c260fb..bb8ddb9dff 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -80,7 +80,7 @@ class BytesCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BytesCodec(CodecEntity): +class BytesCodec(CodecEntity[BytesCodecMetadata]): """The `bytes` codec, coerced from its metadata. `endian` is optional and absent means something: a one-byte data type @@ -119,6 +119,3 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb "missing_key", ) return () - - def to_json(self) -> BytesCodecObject | BytesCodecName: - return cast("BytesCodecObject | BytesCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index adbec3b57d..ad22f71219 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -126,7 +126,7 @@ class CastValueCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class CastValueCodec(CodecEntity): +class CastValueCodec(CodecEntity[CastValueCodecMetadata]): """The `cast_value` codec, coerced from its metadata. Holds the data type it casts to, so like `sharding_indexed` it is @@ -145,6 +145,3 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type return incoming.with_data_type(data_type if isinstance(data_type, DataTypeEntity) else None) - - def to_json(self) -> CastValueCodecObject: - return cast("CastValueCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 96ee88be08..454ff94b35 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -8,7 +8,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -61,7 +61,7 @@ class Crc32cCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class Crc32cCodec(CodecEntity): +class Crc32cCodec(CodecEntity[Crc32cCodecName]): """The `crc32c` codec, coerced from its metadata. The name says everything: a checksum has nothing to configure. @@ -69,6 +69,3 @@ class Crc32cCodec(CodecEntity): identifier: ClassVar[str] = CRC32C_CODEC_NAME kind: ClassVar[CodecKind] = "bytes_bytes" - - def to_json(self) -> Crc32cCodecObject | Crc32cCodecName: - return cast("Crc32cCodecObject | Crc32cCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index cd90ff2372..324fdcd5e7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -66,7 +66,7 @@ class GzipCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class GzipCodec(CodecEntity): +class GzipCodec(CodecEntity[GzipCodecMetadata]): """The `gzip` codec, coerced from its metadata.""" level: Annotated[int, Interval(ge=0, le=9)] @@ -74,6 +74,3 @@ class GzipCodec(CodecEntity): identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" - - def to_json(self) -> GzipCodecObject: - return cast("GzipCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index b7ee90cfad..d9d2ab9ab7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -75,7 +75,7 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ScaleOffsetCodec(CodecEntity): +class ScaleOffsetCodec(CodecEntity[ScaleOffsetCodecMetadata]): """The `scale_offset` codec, coerced from its metadata. Both members are optional and any JSON scalar is well-typed here; what @@ -110,6 +110,3 @@ def _is_a_scalar(value: JSONValue) -> tuple[ValidationProblem, ...]: if value is None: return problem((), "expected a scalar, got null", "invalid_value") return () - - def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: - return cast("ScaleOffsetCodecObject | ScaleOffsetCodecName", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 8d18ffb38d..6a4ceb7465 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -96,7 +96,7 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ShardingIndexedCodec(CodecEntity): +class ShardingIndexedCodec(CodecEntity[ShardingIndexedCodecMetadata]): """The `sharding_indexed` codec, coerced from its metadata. Holds two codec pipelines, so it is one of the few entities that @@ -184,6 +184,3 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation ) ) return tuple(found) - - def to_json(self) -> ShardingIndexedCodecObject: - return cast("ShardingIndexedCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 615e6791a2..2ef20918c6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -64,7 +64,7 @@ class TransposeCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class TransposeCodec(CodecEntity): +class TransposeCodec(CodecEntity[TransposeCodecMetadata]): """The `transpose` codec, coerced from its metadata.""" order: tuple[int, ...] @@ -110,6 +110,3 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: longer the grid the document wrote. """ return incoming.with_grid(incoming.grid.permuted(self.order)) - - def to_json(self) -> TransposeCodecObject: - return cast("TransposeCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 34f9198243..af652a773d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -7,7 +7,7 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict @@ -74,7 +74,7 @@ class ZstdCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ZstdCodec(CodecEntity): +class ZstdCodec(CodecEntity[ZstdCodecMetadata]): """The `zstd` codec, coerced from its metadata.""" level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] @@ -83,6 +83,3 @@ class ZstdCodec(CodecEntity): identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" - - def to_json(self) -> ZstdCodecObject: - return cast("ZstdCodecObject", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index a207141ef3..4645732a62 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -20,6 +20,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( DataTypeEntity, + JSONT_co, StorageClass, is_integer, problem, @@ -63,7 +64,7 @@ def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[Validati @dataclass(frozen=True) -class IntegerDataType(DataTypeEntity, base=True): +class IntegerDataType(DataTypeEntity[JSONT_co], base=True): """A fixed-width integer. The width is the whole difference.""" bounds: ClassVar[tuple[int, int]] @@ -80,7 +81,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class FloatDataType(DataTypeEntity, base=True): +class FloatDataType(DataTypeEntity[JSONT_co], base=True): """A binary float. A fill value may be a number, a named non-finite, or hex.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -120,7 +121,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class ComplexDataType(DataTypeEntity, base=True): +class ComplexDataType(DataTypeEntity[JSONT_co], base=True): """A complex number: a `[real, imag]` pair of the component float type.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -167,7 +168,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class NumpyTimeDataType(DataTypeEntity, base=True): +class NumpyTimeDataType(DataTypeEntity[JSONT_co], base=True): """A numpy time scalar: a signed 64-bit count of units, or `NaT`. The vocabulary the two time types share -- the unit codes and the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index d240ffdf2d..3c49b4349d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -34,7 +34,7 @@ @dataclass(frozen=True) -class BoolDataType(DataTypeEntity): +class BoolDataType(DataTypeEntity[BoolDataTypeName]): """The `bool` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index c52e5237e6..f2d2fa38fe 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -60,7 +60,7 @@ def base64_bytes(value: str) -> Base64Bytes: @dataclass(frozen=True) -class BytesDataType(DataTypeEntity): +class BytesDataType(DataTypeEntity[BytesDataTypeName]): """The `bytes` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "variable_length" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 12ff217385..eb5d554edb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -42,7 +42,7 @@ @dataclass(frozen=True) -class Complex128DataType(ComplexDataType): +class Complex128DataType(ComplexDataType[Complex128DataTypeName]): """The `complex128` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index cf06a9214d..715557a901 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -42,7 +42,7 @@ @dataclass(frozen=True) -class Complex64DataType(ComplexDataType): +class Complex64DataType(ComplexDataType[Complex64DataTypeName]): """The `complex64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index d6504868d4..10cb15ce14 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -81,7 +81,7 @@ def hex_float16(value: str) -> HexFloat16: @dataclass(frozen=True) -class Float16DataType(FloatDataType): +class Float16DataType(FloatDataType[Float16DataTypeName]): """The `float16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 5e2b287d52..46658f93a9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -81,7 +81,7 @@ def hex_float32(value: str) -> HexFloat32: @dataclass(frozen=True) -class Float32DataType(FloatDataType): +class Float32DataType(FloatDataType[Float32DataTypeName]): """The `float32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index b52b9205bd..531407fb07 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -82,7 +82,7 @@ def hex_float64(value: str) -> HexFloat64: @dataclass(frozen=True) -class Float64DataType(FloatDataType): +class Float64DataType(FloatDataType[Float64DataTypeName]): """The `float64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index fb295e3dfc..d4536b2815 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int16DataType(IntegerDataType): +class Int16DataType(IntegerDataType[Int16DataTypeName]): """The `int16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 7d35f033cd..0d57f988c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int32DataType(IntegerDataType): +class Int32DataType(IntegerDataType[Int32DataTypeName]): """The `int32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index 365370a9dc..b957738f45 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int64DataType(IntegerDataType): +class Int64DataType(IntegerDataType[Int64DataTypeName]): """The `int64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 5a2bf185ca..bfecdacb04 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int8DataType(IntegerDataType): +class Int8DataType(IntegerDataType[Int8DataTypeName]): """The `int8` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 26ce5a67ba..b35ec45ce0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict @@ -69,7 +69,7 @@ class NumpyDatetime64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyDatetime64DataType(NumpyTimeDataType): +class NumpyDatetime64DataType(NumpyTimeDataType[NumpyDatetime64]): """The `numpy.datetime64` data type, coerced from its metadata.""" unit: NumpyTimeUnit @@ -77,6 +77,3 @@ class NumpyDatetime64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME - - def to_json(self) -> NumpyDatetime64: - return cast("NumpyDatetime64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 19d3536c49..4e596d13ef 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -5,7 +5,7 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired, cast +from typing import Annotated, ClassVar, Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict @@ -72,7 +72,7 @@ class NumpyTimedelta64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyTimedelta64DataType(NumpyTimeDataType): +class NumpyTimedelta64DataType(NumpyTimeDataType[NumpyTimedelta64]): """The `numpy.timedelta64` data type, coerced from its metadata.""" unit: NumpyTimeUnit @@ -80,6 +80,3 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME - - def to_json(self) -> NumpyTimedelta64: - return cast("NumpyTimedelta64", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 5816870596..f482ba740e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -10,12 +10,11 @@ import re from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, NewType, Self, cast +from typing import Annotated, ClassVar, Final, NewType, Self from typing_extensions import TypedDict, Unpack from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( FROM_NAME, Coerced, @@ -107,7 +106,7 @@ class RawBytesMembers(TypedDict): @dataclass(frozen=True) -class RawBytesDataType(DataTypeEntity): +class RawBytesDataType(DataTypeEntity[RawBytesDataTypeName]): """An `r` raw-bytes data type, coerced from its metadata. One class for the whole family, because `r8` and `r4096` differ only @@ -163,8 +162,8 @@ def value_problems(**members: Unpack[RawBytesMembers]) -> tuple[ValidationProble """ return _name_problems(members["data_type_name"]) - def to_json(self) -> ZarrV3MetadataFieldJSON: - return cast("ZarrV3MetadataFieldJSON", self.data_type_name) + def to_json(self) -> RawBytesDataTypeName: + return RawBytesDataTypeName(self.data_type_name) def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """One byte value per byte of the scalar. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 826273607c..5000788e64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -34,7 +34,7 @@ @dataclass(frozen=True) -class StringDataType(DataTypeEntity): +class StringDataType(DataTypeEntity[StringDataTypeName]): """The `string` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "variable_length" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 1db7adf145..bcc4b8191b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -97,7 +97,7 @@ class StructFieldComponent: @dataclass(frozen=True) -class StructDataType(DataTypeEntity): +class StructDataType(DataTypeEntity[Struct]): """The `struct` data type, coerced from its metadata. A record of named fields, each with a data type of its own -- so this @@ -207,6 +207,3 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP for key in sorted(fills.keys() - declared) ) return tuple(found) - - def to_json(self) -> Struct: - return cast("Struct", super().to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index d87340123c..19055a172a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint16DataType(IntegerDataType): +class Uint16DataType(IntegerDataType[Uint16DataTypeName]): """The `uint16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index ab0c7b2cce..d07bc8ce32 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint32DataType(IntegerDataType): +class Uint32DataType(IntegerDataType[Uint32DataTypeName]): """The `uint32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 241f4eb819..89b2816b3b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint64DataType(IntegerDataType): +class Uint64DataType(IntegerDataType[Uint64DataTypeName]): """The `uint64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index fb5fabbf5e..aa9b8a04d9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint8DataType(IntegerDataType): +class Uint8DataType(IntegerDataType[Uint8DataTypeName]): """The `uint8` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 49d810ff31..64e689676a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -26,7 +26,9 @@ **Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, `ChunkGridEntity` or `MetadataEntity`, declare the fields, and add it to -a scope: +a scope. Name your JSON type as the base's argument if you have one -- +`CodecEntity[AcmeLz4Metadata]` -- and `to_json` is typed as it; left +bare, `to_json` is typed as any metadata field: @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index d05b45dcd6..f4e8b03be1 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,38 +11,52 @@ import copy import dataclasses -from typing import Any, ClassVar, Self, cast, get_args, get_type_hints +import types +from typing import ( + Any, + ClassVar, + NotRequired, + Self, + Union, + cast, + get_args, + get_origin, + get_type_hints, +) import pytest +from hypothesis import given, settings +from typing_extensions import ReadOnly, is_typeddict +from tests.rules.strategies import valid_documents from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3._compile import check_for +from zarr_metadata.v3._document import read_array_v3 +from zarr_metadata.v3._entity import json_type_of from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, - RectilinearChunkGridConfiguration, ) -from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid, RegularChunkGridConfiguration +from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid from zarr_metadata.v3.chunk_key_encoding.default import ( DefaultChunkKeyEncoding, - DefaultChunkKeyEncodingConfiguration, ) from zarr_metadata.v3.chunk_key_encoding.v2 import ( V2ChunkKeyEncoding, - V2ChunkKeyEncodingConfiguration, ) -from zarr_metadata.v3.codec.blosc import BloscCodec, BloscCodecConfiguration -from zarr_metadata.v3.codec.bytes import BytesCodec, BytesCodecConfiguration -from zarr_metadata.v3.codec.cast_value import CastValueCodec, CastValueCodecConfiguration -from zarr_metadata.v3.codec.crc32c import Crc32cCodec, Empty -from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecConfiguration -from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec, ScaleOffsetCodecConfiguration +from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.codec.bytes import BytesCodec +from zarr_metadata.v3.codec.cast_value import CastValueCodec +from zarr_metadata.v3.codec.crc32c import Crc32cCodec +from zarr_metadata.v3.codec.gzip import GzipCodec +from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec from zarr_metadata.v3.codec.sharding_indexed import ( ShardingIndexedCodec, - ShardingIndexedCodecConfiguration, ) -from zarr_metadata.v3.codec.transpose import TransposeCodec, TransposeCodecConfiguration -from zarr_metadata.v3.codec.zstd import ZstdCodec, ZstdCodecConfiguration +from zarr_metadata.v3.codec.transpose import TransposeCodec +from zarr_metadata.v3.codec.zstd import ZstdCodec from zarr_metadata.v3.data_type.bool import BoolDataType from zarr_metadata.v3.data_type.bytes import BytesDataType from zarr_metadata.v3.data_type.complex64 import Complex64DataType @@ -55,16 +69,14 @@ from zarr_metadata.v3.data_type.int32 import Int32DataType from zarr_metadata.v3.data_type.int64 import Int64DataType from zarr_metadata.v3.data_type.numpy_datetime64 import ( - NumpyDatetime64Configuration, NumpyDatetime64DataType, ) from zarr_metadata.v3.data_type.numpy_timedelta64 import ( - NumpyTimedelta64Configuration, NumpyTimedelta64DataType, ) from zarr_metadata.v3.data_type.raw import RawBytesDataType from zarr_metadata.v3.data_type.string import StringDataType -from zarr_metadata.v3.data_type.struct import StructConfiguration, StructDataType +from zarr_metadata.v3.data_type.struct import StructDataType from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType @@ -110,48 +122,29 @@ "data_type:r": RawBytesDataType, } -# The public JSON TypedDict each configured entity's fields must mirror. Test -# data, not a class attribute: nothing in the package reads it any more, so -# this is the one correspondence still written by hand -- and the one that -# catches an entity whose fields drift from the JSON type it is documented by. -# An entity absent here has no configuration. -CONFIGURATIONS: dict[str, type] = { - "codecs:blosc": BloscCodecConfiguration, - "codecs:bytes": BytesCodecConfiguration, - "codecs:cast_value": CastValueCodecConfiguration, - "codecs:crc32c": Empty, - "codecs:gzip": GzipCodecConfiguration, - "codecs:scale_offset": ScaleOffsetCodecConfiguration, - "codecs:sharding_indexed": ShardingIndexedCodecConfiguration, - "codecs:transpose": TransposeCodecConfiguration, - "codecs:zstd": ZstdCodecConfiguration, - "chunk_grid:regular": RegularChunkGridConfiguration, - "chunk_grid:rectilinear": RectilinearChunkGridConfiguration, - "chunk_key_encoding:default": DefaultChunkKeyEncodingConfiguration, - "chunk_key_encoding:v2": V2ChunkKeyEncodingConfiguration, - "data_type:numpy.datetime64": NumpyDatetime64Configuration, - "data_type:numpy.timedelta64": NumpyTimedelta64Configuration, - "data_type:struct": StructConfiguration, -} - @pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) def test_the_constructor_mirrors_the_configuration(entity: type[MetadataEntity]) -> None: # The member table and `configuration_required` are read off the fields, - # so the fields are the only spelling left that can drift from the public - # JSON TypedDict -- and a field the TypedDict does not have would be a - # member no document could write. - # - # `must_understand` belongs to the object, not the configuration, so it - # is the one field the two deliberately do not share. - key = next(key for key, candidate in ENTITIES.items() if candidate is entity) + # and the JSON type is named as the base's argument; the fields are the + # only spelling left that can drift from the public TypedDict -- and a + # field the TypedDict does not have would be a member no document could + # write. `must_understand` belongs to the object, not the configuration, + # so it is the one field the two deliberately do not share. fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} - if key not in CONFIGURATIONS: - # `r` keeps its width in its name, so it holds a member that - # is not a configuration key. + json_type = json_type_of(entity) + objects = [part for part in _parts(json_type) if is_typeddict(part)] + if len(objects) == 0: + # A bare-name type: nothing to configure. `r` keeps its width + # in its name, so it holds a member that is not a configuration key. assert fields == ({"data_type_name"} if entity is RawBytesDataType else set()) return - assert fields == set(get_type_hints(CONFIGURATIONS[key])) + (obj,) = objects + configuration = get_type_hints(obj, include_extras=True).get("configuration") + assert configuration is not None, f"{obj!r} has no configuration member" + while get_origin(configuration) in (NotRequired, ReadOnly): + (configuration,) = get_args(configuration) + assert fields == set(get_type_hints(configuration)) @pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) @@ -173,6 +166,165 @@ def test_the_value_routine_takes_the_members_it_will_be_given( assert set(get_type_hints(members)) == fields +def _parts(json_type: object) -> tuple[object, ...]: + return ( + get_args(json_type) if get_origin(json_type) in (Union, types.UnionType) else (json_type,) + ) + + +@pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) +def test_every_entity_names_its_json_type(entity: type[MetadataEntity]) -> None: + # The default is not wrong, only uninformative; every entity this + # package models says exactly what it writes. + assert json_type_of(entity) is not ZarrV3MetadataFieldJSON + + +# One or more documents each entity reads, spelled to reach both shapes +# where the entity has both: the bare name when every member is absent, +# the object otherwise. `st.from_type` over the named types cannot serve +# here -- measured, it reaches a valid gzip or blosc in under 1% of draws. +EXAMPLES: dict[str, tuple[object, ...]] = { + "codecs:blosc": ( + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "typesize": 4, + "blocksize": 0, + }, + }, + ), + "codecs:bytes": ("bytes", {"name": "bytes", "configuration": {"endian": "little"}}), + "codecs:cast_value": ( + {"name": "cast_value", "configuration": {"data_type": "int8"}}, + { + "name": "cast_value", + "configuration": {"data_type": "int8", "scalar_map": {"encode": (("NaN", 0),)}}, + }, + ), + "codecs:crc32c": ("crc32c", {"name": "crc32c"}), + "codecs:gzip": ({"name": "gzip", "configuration": {"level": 5}},), + "codecs:scale_offset": ( + "scale_offset", + {"name": "scale_offset", "configuration": {"offset": 2, "scale": 0.5}}, + ), + "codecs:sharding_indexed": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4,), + "codecs": ("bytes",), + "index_codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + "crc32c", + ), + "index_location": "start", + }, + }, + ), + "codecs:transpose": ({"name": "transpose", "configuration": {"order": (2, 1, 0)}},), + "codecs:zstd": ({"name": "zstd", "configuration": {"level": 3, "checksum": False}},), + "chunk_grid:regular": ({"name": "regular", "configuration": {"chunk_shape": (4, 4)}},), + "chunk_grid:rectilinear": ( + { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": ((32, 32, 32),)}, + }, + { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": (((32, 3),),)}, + }, + ), + "chunk_key_encoding:default": ( + "default", + {"name": "default", "configuration": {"separator": "."}}, + ), + "chunk_key_encoding:v2": ("v2", {"name": "v2", "configuration": {"separator": "/"}}), + "data_type:numpy.datetime64": ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, + ), + "data_type:numpy.timedelta64": ( + {"name": "numpy.timedelta64", "configuration": {"unit": "ms", "scale_factor": 10}}, + ), + "data_type:bool": ("bool",), + "data_type:int8": ("int8",), + "data_type:int16": ("int16",), + "data_type:int32": ("int32",), + "data_type:int64": ("int64",), + "data_type:uint8": ("uint8",), + "data_type:uint16": ("uint16",), + "data_type:uint32": ("uint32",), + "data_type:uint64": ("uint64",), + "data_type:float16": ("float16",), + "data_type:float32": ("float32",), + "data_type:float64": ("float64",), + "data_type:complex64": ("complex64",), + "data_type:complex128": ("complex128",), + "data_type:bytes": ("bytes",), + "data_type:struct": ( + { + "name": "struct", + "configuration": { + "fields": ( + {"name": "a", "data_type": "uint8"}, + { + "name": "b", + "data_type": { + "name": "numpy.datetime64", + "configuration": {"unit": "s", "scale_factor": 1}, + }, + }, + ), + }, + }, + ), + "data_type:string": ("string",), + "data_type:r": ("r16", "r008"), +} + + +def _assert_conforms(entity: MetadataEntity) -> None: + # The one `cast` in `to_json` asserts that what it builds has the + # entity's named type. This is that assertion, checked: the named + # type compiled by the package's own compiler, and the output run + # through it. + json_type = json_type_of(type(entity)) + check = check_for(json_type) + assert check is not None, f"{json_type!r} is not a shape the compiler reads" + assert check(entity.to_json(), ()) == () + + +@pytest.mark.parametrize( + ("entity", "document"), + [(ENTITIES[key], document) for key, documents in EXAMPLES.items() for document in documents], + ids=[ + f"{key}:{index}" for key, documents in EXAMPLES.items() for index in range(len(documents)) + ], +) +def test_to_json_conforms_to_the_named_json_type( + entity: type[MetadataEntity], document: object +) -> None: + read, problems = entity.coerce(document, CORE_AND_EXTENSIONS) + assert problems == () + assert read is not None + _assert_conforms(read) + + +@given(document=valid_documents()) +@settings(max_examples=50, deadline=None) +def test_to_json_conforms_across_a_valid_document(document: dict[str, object]) -> None: + # The top-level entities of documents valid by construction, for the + # variation the examples fix: permutations, chunk shapes, an index + # pipeline. A nested entity is some entity's top-level example. + array, problems = read_array_v3(document, CORE_AND_EXTENSIONS) + assert problems == () + for entity in (array.data_type, array.chunk_grid, array.chunk_key_encoding, *array.codecs): + assert isinstance(entity, MetadataEntity) + _assert_conforms(entity) + + def test_every_registered_entity_is_checked_here() -> None: registered = { f"{field}:{identifier}" @@ -180,6 +332,7 @@ def test_every_registered_entity_is_checked_here() -> None: for identifier in entities } assert registered == set(ENTITIES) + assert set(EXAMPLES) == set(ENTITIES) def test_core_is_a_subset_of_core_and_extensions() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index eff7e20b9d..0cfd129c39 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,16 +8,19 @@ import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, ClassVar, NewType, Self, cast +from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast import pytest +from typing_extensions import TypedDict from zarr_metadata.model import UNSET, MetadataValidationError, ValidationProblem from zarr_metadata.rules import ( canonicalize_array_metadata_v3, validate_array_metadata_v3, ) +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._compile import _CHECK_COMPILERS +from zarr_metadata.v3._entity import json_type_of from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.entity import ( @@ -48,10 +51,6 @@ ACME_MAX_ACCELERATION = 65537 -if TYPE_CHECKING: - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - - @dataclass(frozen=True) class AcmeLz4Codec(CodecEntity): """A third-party compressor.""" @@ -567,7 +566,10 @@ def _rule(block: int) -> tuple[ValidationProblem, ...]: # An annotation shape the compiler does not read, taught to it from outside. -Hex = NewType("Hex", str) +class Hex(str): + """A hex digest: a `str` to the type checker, its own class at run time.""" + + __slots__ = () def _is_hex(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: @@ -577,10 +579,10 @@ def _is_hex(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: def test_a_third_party_can_teach_the_compiler_a_shape() -> None: - # `NewType` is a real case: to the type checker `Hex` is a `str`, but - # at run time it is a function `check_for` has no registration for, - # so an entity using it is refused -- until one is registered, through - # the same door the built-in shapes came through. + # A `str` subclass is a real case: to the type checker `Hex` is a + # `str`, but at run time it is a class `check_for` has no registration + # for, so an entity using it is refused -- until one is registered, + # through the same door the built-in shapes came through. with pytest.raises(TypeError, match="no check can be read off the annotation of digest"): @dataclass(frozen=True) @@ -622,3 +624,43 @@ class AcmeDigestCodec(CodecEntity): _CHECK_COMPILERS[:] = [ entry for entry in _CHECK_COMPILERS if entry.predicate is not is_hex_annotation ] + + +def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: + # A required member means the entity is always written as an object, + # so naming a bare-name type for it is a promise `to_json` would break. + with pytest.raises(TypeError, match="lacks an object, but the entity never writes a bare name"): + + @dataclass(frozen=True) + class Misnamed(CodecEntity[Literal["acme.misnamed"]]): # pyright: ignore[reportUnusedClass] + level: int + + identifier: ClassVar[str] = "acme.misnamed" + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_a_third_party_entity_may_name_its_json_type_or_not() -> None: + # Left defaulted, `to_json` is typed as any metadata field; named, as + # the entity's own type -- and either way the same dict comes back. + assert json_type_of(AcmeLz4Codec) is ZarrV3MetadataFieldJSON + + class AcmeBlockConfiguration(TypedDict, closed=True): + block: int + + class AcmeBlockObject(TypedDict, closed=True): + name: Literal["acme.block"] + configuration: AcmeBlockConfiguration + must_understand: NotRequired[bool] + + @dataclass(frozen=True) + class AcmeTypedBlockCodec(CodecEntity[AcmeBlockObject]): + block: int + + identifier: ClassVar[str] = "acme.block" + kind: ClassVar[CodecKind] = "bytes_bytes" + + assert json_type_of(AcmeTypedBlockCodec) is AcmeBlockObject + assert AcmeTypedBlockCodec(block=8).to_json() == { + "name": "acme.block", + "configuration": {"block": 8}, + } From 64313203dab0942c9b4dd585619221f7be1e8136 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 12:22:09 +0200 Subject: [PATCH 069/107] refactor(zarr-metadata): canonical is the walk, then the entity's own rewrite `canonical` walks into contained entities and then calls `simplified`, the hook where an entity says two spellings of its own members mean the same; blosc, rectilinear and the test's storage transformer move their rewrites there. Overriding `canonical` is refused at class creation, so the walk cannot be lost and there is no `super()` to remember -- the three overrides were calling it for nothing, since none of them contains an entity. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 7 +++ .../src/zarr_metadata/v3/_entity.py | 54 ++++++++++++++----- .../v3/chunk_grid/rectilinear.py | 4 +- .../src/zarr_metadata/v3/codec/blosc.py | 9 ++-- .../zarr-metadata/tests/v3/test_entities.py | 4 +- .../tests/v3/test_extension_api.py | 39 +++++++++++++- 6 files changed, 93 insertions(+), 24 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 5e04a3d35d..f5e37f44c1 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -67,6 +67,13 @@ three `canonical` overrides that did that walk by hand for `cast_value`, field typed as bare `MetadataEntity` is refused at class creation, since no scope could place it. +That walk is `canonical`'s alone. An entity's own rewrite -- two +spellings of its own members that mean the same, a rectilinear +dimension's run-length encoding, a `typesize` that `noshuffle` ignores +-- goes in `simplified`, a hook `canonical` calls after the walk; an +override of `canonical` itself is refused at class creation, so the +walk cannot be lost and there is no `super()` to remember. + A bound on a value is written on the field, in the `annotated_types` vocabulary -- `level: Annotated[int, Interval(ge=0, le=9)]`, `chunk_shape: tuple[Annotated[int, Ge(1)], ...]` -- and judged at diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index cc0abf14e9..e31bdaacc9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -627,6 +627,16 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: "from its field annotation, and nothing calls `prepare`" ) raise TypeError(msg) + if "canonical" in cls.__dict__: + # The walk into contained entities is read off the annotations + # and must not be lost under an override; the entity's own + # rewrite has a hook of its own. + msg = ( + f"{cls.__name__} overrides `canonical`, which is the walk into contained " + "entities; put the entity's own rewrite in `simplified`, which " + "`canonical` calls after the walk" + ) + raise TypeError(msg) if "__post_init__" in cls.__dict__: # `coerce` builds through `unchecked`, which bypasses # `__init__` and so never reaches `__post_init__`. Rules put @@ -859,23 +869,39 @@ def canonical(self) -> Self: a reader that reads and writes should not change bytes it was not asked to change. - A contained entity is put in its own canonical form here, by - walking the fields that hold one. Override where two spellings of - the entity's *own* members mean the same -- a rectilinear - dimension's run-length encoding, a `typesize` that `noshuffle` - ignores -- and start from `super().canonical()`, so the walk is - not lost. + Two steps, each with one owner. Every contained entity is put in + its own canonical form by walking the fields that hold one, which + is read off the annotations and is this method's alone -- an + override of it is refused at class creation. Then `simplified`, + the entity's own rewrite, which is where an entity says that two + spellings of its own members mean the same. """ nested = type(self).nested_members - if len(nested) == 0: - return self - return replace( - self, - **{ - name: canonicalize_nested(annotation, getattr(self, name)) - for name, annotation in nested.items() - }, + walked = ( + self + if len(nested) == 0 + else replace( + self, + **{ + name: canonicalize_nested(annotation, getattr(self, name)) + for name, annotation in nested.items() + }, + ) ) + return walked.simplified() + + def simplified(self) -> Self: + """This entity with its own members in their simplest equivalent spelling. + + The hook `canonical` calls once every contained entity is in + canonical form. Override it where two spellings of the entity's + *own* members mean the same -- a rectilinear dimension's + run-length encoding, a `typesize` that `noshuffle` ignores -- and + return the entity rewritten. The default is the identity, and + there is nothing to call `super()` for: the walk into contained + entities is not this method's to keep. + """ + return self def configuration(self) -> dict[str, object]: """This entity's configuration, as the document would write it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index f86ac9268f..ad59d3da2b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -221,10 +221,10 @@ def grid(self, array_shape: object) -> ChunkGrid: """ return ChunkGrid.derived(tuple(_axis_lengths(spec) for spec in self.chunk_shapes)) - def canonical(self) -> Self: + def simplified(self) -> Self: """Run-length encoded, which is the spelling that does not grow. Two dimension specs listing the same extents describe the same grid, and the encoded one stays the same size as the array grows. """ - return replace(super().canonical(), chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) + return replace(self, chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index f29685c262..f2480f8f7c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -142,13 +142,12 @@ def value_problems( ) return tuple(found) - def canonical(self) -> Self: + def simplified(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. The spec says of that case that "the value is ignored", so two documents differing only there describe the same codec. """ - canonical = super().canonical() - if canonical.shuffle != BLOSC_NO_SHUFFLE or canonical.typesize is UNSET: - return canonical - return replace(canonical, typesize=UNSET) + if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: + return self + return replace(self, typesize=UNSET) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index f4e8b03be1..5833a627ea 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -744,8 +744,8 @@ class AcmeShardCache(MetadataEntity): identifier: ClassVar[str] = "acme.shard_cache" - def canonical(self) -> Self: - return dataclasses.replace(super().canonical(), verbose=UNSET) + def simplified(self) -> Self: + return dataclasses.replace(self, verbose=UNSET) def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 0cfd129c39..badc46575b 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast import pytest @@ -496,6 +496,43 @@ def prepare(cls, members: object, context: object) -> object: return members +def test_error_an_entity_may_not_override_canonical() -> None: + # `canonical` is the walk into contained entities, read off the + # annotations; an override could lose it. The entity's own rewrite + # goes in `simplified`. + with pytest.raises(TypeError, match="put the entity's own rewrite in `simplified`"): + + @dataclass(frozen=True) + class Rewriter(CodecEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.rewriter" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def canonical(self) -> Self: + return self + + +def test_simplified_composes_with_the_walk_into_contained_entities() -> None: + # An entity that contains an entity and rewrites its own members gets + # both from `canonical` -- the contained blosc loses the `typesize` + # that `noshuffle` ignores, and the frame of 0 that means "unframed" + # is dropped -- with nothing to call `super()` for. + @dataclass(frozen=True) + class AcmeFramedCodec(CodecEntity): + inner: CodecEntity | Opaque + frame: int | UNSET = UNSET + + identifier: ClassVar[str] = "acme.framed" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def simplified(self) -> Self: + return self if self.frame != 0 else replace(self, frame=UNSET) + + blosc = BloscCodec(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) + framed = AcmeFramedCodec(inner=blosc, frame=0) + assert framed.canonical() == AcmeFramedCodec(inner=replace(blosc, typesize=UNSET)) + assert framed.inner is blosc # a transformation, not a mutation + + def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: # `MetadataEntity` is registered at no single point, so a field typed # as one could not be resolved through any scope. From fa61fffc5830a48f1c9995912467b3bcf5133850 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 12:25:02 +0200 Subject: [PATCH 070/107] refactor(zarr-metadata): the member table is compiled, never declared `member_types` is the checks read off the fields, consulted by `coerce` member by member; the by-hand entry for an annotation the compiler could not read was a second door for what `register_check` does once per shape, and it carried a guard of its own against restating requiredness. Both go: an unread annotation is refused at class creation with a pointer to `register_check`. The class docstring stops saying that subclasses declare the table, and that an optional member is typed `| None` -- it is `| UNSET`, and has been. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 13 ++--- .../src/zarr_metadata/v3/_entity.py | 48 +++++++------------ .../src/zarr_metadata/v3/entity.py | 7 +-- .../tests/v3/test_extension_api.py | 19 +------- 4 files changed, 28 insertions(+), 59 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index f5e37f44c1..ab4b579324 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -32,12 +32,13 @@ written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field annotations: `int`, `bool`, `str`, a `Literal` of names, the JSON-value alias, an array homogeneous or fixed, a union of those, a nested object -described by a TypedDict or a record dataclass, and a nested metadata -field -- an entity type, with or without `Opaque`. That covers every -member this package models; `member_types` remains only for an -annotation the compiler does not read, and `Annotated[str, FROM_NAME]` -marks the one field carried by the envelope's name rather than a -configuration key (`r`). `configuration_required` follows too, since +described by a TypedDict or a record dataclass, an object of undeclared +keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested +metadata field -- an entity type, with or without `Opaque`. That covers +every member this package models; an annotation the compiler does not +read is refused at class creation until `register_check` teaches it the +shape, and `Annotated[str, FROM_NAME]` marks the one field carried by +the envelope's name rather than a configuration key (`r`). `configuration_required` follows too, since the spec ties the bare-name spelling to whether any member is required. The public JSON TypedDict is no longer read by the package at all: the fields are held to its keys by a test, which is the one correspondence diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index e31bdaacc9..547a90abd2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -95,7 +95,6 @@ field_hints, has_shape, is_class_var, - is_optional, is_union, own_annotations, register_check, @@ -498,8 +497,9 @@ class MetadataEntity(Generic[JSONT_co]): Subclasses add their configuration members as fields, which is what makes them well-typed by construction: an instance exists only if `coerce` accepted the metadata that produced it. An optional member is - typed `| None` with a default of `None`, so absence is representable - and a canonical spelling can leave it out. + typed `| UNSET` with a default of `UNSET`, so absence is representable + -- and distinct from a `null` the document wrote -- and a canonical + spelling can leave it out. Frozen, so an entity of hashable members is hashable. One holding a value out of scope is not, because that value is the JSON the document @@ -508,11 +508,12 @@ class MetadataEntity(Generic[JSONT_co]): mapping instead: `MappingProxyType` is unhashable too, and anything else stops `json.dumps` from serializing what `to_json` returns. - Most subclasses declare `member_types` and nothing else: the default - `coerce` and `to_json` are written once here against that table. The - ones that override are the ones with something particular to say -- - a configuration containing other entities, a name that is a family - rather than a constant, a member another member renders meaningless. + A subclass writes its fields, and a rule where the spec has something + to say beyond their types. `coerce`, `configuration`, `to_json` and + `canonical` are written once here against what the fields say, and + the class variables below -- `member_types`, `nested_members`, + `value_checks`, `member_rules` -- are that reading, compiled at class + creation: nothing declares them. """ extension_point: ClassVar[ExtensionPointField | None] = None @@ -554,8 +555,10 @@ class MetadataEntity(Generic[JSONT_co]): Read off the dataclass fields at class creation: which members there are, which may be absent (the type admits `UNSET`), and the check - each one's type implies. A class declares an entry itself only for a - field whose annotation `check_for` cannot compile, and the public + each one's type implies. `coerce` reads a configuration against it + member by member, so one member that cannot be read costs that member + and not the rest. An annotation the compiler cannot read is refused + at class creation; `register_check` teaches it the shape. The public JSON TypedDict is held to the same keys by `tests/v3/test_entities.py`. """ @@ -655,32 +658,13 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: ) raise TypeError(msg) # Before every guard below, because they read the table. - declared = dict(vars(cls).get("member_types", {})) - derived, unread = derive_member_types(cls) - unsupported = sorted(set(unread) - set(declared)) - if len(unsupported) != 0: + cls.member_types, unread = derive_member_types(cls) + if len(unread) != 0: msg = ( f"{cls.__name__}: no check can be read off the annotation of " - f"{', '.join(unsupported)}; declare one in `member_types`" + f"{', '.join(sorted(unread))}; teach the compiler that shape with `register_check`" ) raise TypeError(msg) - optional = {name: is_optional(annotation) for name, annotation in field_hints(cls).items()} - misstated = sorted( - member - for member, (required, _) in declared.items() - if member in optional and required == optional[member] - ) - if len(misstated) != 0: - # The check is the entity's to write; whether the member may - # be left out is the field's to say, and a declared entry - # that disagrees is the drift this derivation exists to rule - # out. - msg = ( - f"{cls.__name__} declares {', '.join(misstated)} with a requiredness " - "its field does not give it" - ) - raise TypeError(msg) - cls.member_types = {**derived, **declared} cls.configuration_required = any(required for required, _ in cls.member_types.values()) json_type = json_type_of(cls) if json_type is not ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 64e689676a..e49198ad14 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -48,9 +48,10 @@ class AcmeLz4Codec(CodecEntity): The fields are the only place the shape is written. Which members exist, which may be left out (the type admits `UNSET`), and how each one is type-checked are all read off the annotations -- an `int`, a `Literal` -of names, an array, a nested entity type -- and `member_types` is for -the exception, an annotation the compiler does not read. A bound on a -value is written on the field too, in the `annotated_types` vocabulary: +of names, an array, a nested entity type -- and an annotation the +compiler does not read is taught to it once, with `register_check`. A +bound on a value is written on the field too, in the `annotated_types` +vocabulary: acceleration: Annotated[int, Interval(ge=1, le=65537)] | UNSET = UNSET diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index badc46575b..9c02aecf3e 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -37,11 +37,9 @@ IntegerDataType, Interval, Loc, - MemberTypes, MetadataEntity, Opaque, StorageClass, - is_int, named_configuration, problem, register_check, @@ -367,26 +365,11 @@ def test_a_third_party_can_register_a_family() -> None: assert scope.resolve("data_type", "acme.fixed") is None -def test_error_requiredness_may_not_be_restated() -> None: - # It is the field's to say. A declared entry exists for the check, - # which the annotation does not imply; saying the member is required - # as well is the drift the derivation removes. - with pytest.raises(TypeError, match="requiredness its field does not give it"): - - @dataclass(frozen=True) - class Insistent(CodecEntity): # pyright: ignore[reportUnusedClass] - acceleration: int | UNSET = UNSET - - identifier: ClassVar[str] = "acme.insistent" - kind: ClassVar[CodecKind] = "bytes_bytes" - member_types: ClassVar[MemberTypes] = {"acceleration": (True, is_int)} - - def test_error_a_member_needs_a_check_from_somewhere() -> None: # An annotation outside the shapes `check_for` compiles implies no # check, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. - with pytest.raises(TypeError, match="no check can be read off the annotation of inner"): + with pytest.raises(TypeError, match="annotation of inner; teach the compiler that shape"): @dataclass(frozen=True) class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] From e852d1a0a3fa769051f9f619701199204ef07701 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 12:34:47 +0200 Subject: [PATCH 071/107] refactor(zarr-metadata): class creation is a compile step and a list of invariants `__init_subclass__` was thirteen hand-ordered guards in one method, two of them refusing names from this branch's own earlier drafts. Now `_compile_entity` derives the tables from the fields, and `_INVARIANTS` is a declared tuple of named functions, each returning why the compiled class is refused or None. `canonical` and `__post_init__` are `@final`, which pyright checks in the author's editor and the first invariant checks at run time; declaring any derived class variable is refused, where before only `configuration_required` was; the `problems` and `prepare` guards, tombstones of hooks that never shipped, are gone. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 10 + .../src/zarr_metadata/v3/_entity.py | 434 ++++++++++-------- .../tests/v3/test_extension_api.py | 96 +--- 3 files changed, 272 insertions(+), 268 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 7929b39123..9a8e767a5d 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -39,6 +39,16 @@ it did not: `Mapping[str, V]`, an object of undeclared keys, and a `NewType`, as the type it names -- which also closes a gap for a third-party field typed with either. +Class creation is two steps with one owner each: `_compile_entity` +derives the tables the layer reads from the fields, and `_INVARIANTS`, +a declared tuple of named functions, asks each invariant of the +compiled class in order. `canonical` and `__post_init__` are `@final`, +so an override is refused by pyright in the author's editor as well as +at class creation; the guards that refused `problems` and `prepare` -- +names from earlier drafts of this branch, never released -- are gone, +and declaring any derived class variable is refused, where before only +`configuration_required` was. + One thing this does not change, under mypy. An entity's JSON type is a TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as `Mapping[str, object]`, never as the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 547a90abd2..defde4411f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -42,6 +42,7 @@ Literal, TypeAlias, cast, + final, get_args, get_origin, get_type_hints, @@ -490,6 +491,237 @@ class Opaque: # have to spell it `super(Cls, self)`. CPython fixed this in 3.13, so when # that is the floor this is worth revisiting; the memory saved is small at # document scale, which is why it has not been. +_DERIVED: Final = ( + "member_types", + "configuration_required", + "nested_members", + "value_checks", + "member_rules", +) +"""The class variables `_compile_entity` derives; a declaration of one is refused.""" + + +def _compile_entity(cls: type[MetadataEntity]) -> None: + """Derive from the fields the tables the layer reads. + + Raises for a declaration that cannot be compiled: a derived table + declared by hand, which the derivation would silently overwrite; a + field annotation no registered shape reads; a `@validates` naming no + field. + """ + declared = [name for name in _DERIVED if name in vars(cls)] + if len(declared) != 0: + msg = ( + f"{cls.__name__} declares {', '.join(declared)}, which is derived from the " + "fields at class creation" + ) + raise TypeError(msg) + cls.member_types, unread = derive_member_types(cls) + if len(unread) != 0: + msg = ( + f"{cls.__name__}: no check can be read off the annotation of " + f"{', '.join(sorted(unread))}; teach the compiler that shape with `register_check`" + ) + raise TypeError(msg) + cls.configuration_required = any(required for required, _ in cls.member_types.values()) + hints = field_hints(cls) + cls.nested_members = { + name: annotation for name, annotation in hints.items() if contains_entity(annotation) + } + cls.value_checks = { + name: check + for name, annotation in hints.items() + if (check := value_check_for(annotation)) is not None + } + cls.member_rules = _member_rules(cls, hints) + + +def _member_rules( + cls: type[MetadataEntity], hints: Mapping[str, object] +) -> dict[str, tuple[MemberRule, ...]]: + """The `@validates` rules in the class and its ancestors, by the member each is about.""" + attributes: dict[str, object] = {} + for ancestor in reversed(cls.__mro__): + attributes.update(vars(ancestor)) + rules: dict[str, list[MemberRule]] = {} + for attribute in attributes.values(): + function = attribute.__func__ if isinstance(attribute, staticmethod) else attribute + # Only a function can carry the mark; a table-valued class + # attribute is not even hashable. + if not callable(function): + continue + for member in rule_members(function): + if member not in hints: + msg = f"{cls.__name__}: `@validates({member!r})` names no field of the entity" + raise TypeError(msg) + rules.setdefault(member, []).append(cast("MemberRule", function)) + return {member: tuple(found) for member, found in rules.items()} + + +# The invariants, each a function of the compiled class returning why it +# is refused, or None. Every one names something that type-checks cleanly +# and then goes wrong somewhere that will not name the class. + +_FINAL_ADVICE: Final[Mapping[str, str]] = { + "canonical": ( + "which is the walk into contained entities; put the entity's own rewrite " + "in `simplified`, which `canonical` calls after the walk" + ), + "__post_init__": ( + "which `unchecked` does not reach, and `coerce` builds through `unchecked`; " + "value rules belong in `value_problems`" + ), +} +"""What to do instead, for each method the base marks `@final`.""" + + +def _final_methods_are_not_overridden(cls: type[MetadataEntity]) -> str | None: + # `@final` is a promise pyright checks in the author's editor; this + # is the same promise for a class created without one. + for name in vars(cls): + if getattr(getattr(MetadataEntity, name, None), "__final__", False): + return f"{cls.__name__} overrides `{name}`, {_FINAL_ADVICE.get(name, 'which is final')}" + return None + + +def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | None: + # The named type is a promise about what `to_json` writes, and its + # shape follows from the members: a bare name only when no member is + # required and the entity must be understood, an object whenever + # there is a member to write or the flag to. + json_type = json_type_of(cls) + if json_type is ZarrV3MetadataFieldJSON: + return None + admits_bare, admits_object = _json_shape(json_type) + writes_bare = not cls.configuration_required and cls.must_understand + writes_object = len(cls.member_types) != 0 or not cls.must_understand + if admits_bare == writes_bare and admits_object == writes_object: + return None + return ( + f"{cls.__name__} names {json_type!r} as its JSON type, which " + f"{'admits' if admits_bare else 'lacks'} a bare name and " + f"{'admits' if admits_object else 'lacks'} an object, but the entity " + f"{'writes' if writes_bare else 'never writes'} a bare name and " + f"{'writes' if writes_object else 'never writes'} an object" + ) + + +def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: + # `MetadataEntity` itself is registered at no single point, so a + # field typed as one could not be resolved through a scope. + unplaced = sorted( + name + for name, annotation in cls.nested_members.items() + if any(kind.extension_point is None for kind in _entity_kinds(annotation)) + ) + if len(unplaced) == 0: + return None + return ( + f"{cls.__name__}: the entity kind of {', '.join(unplaced)} has no " + "`extension_point`; annotate it with `CodecEntity`, `DataTypeEntity` " + "or `ChunkGridEntity`" + ) + + +def _fields_do_not_shadow_class_variables(cls: type[MetadataEntity]) -> str | None: + # A field of that name would go into `member_types`, into the + # configuration, and into the JSON -- while the class variable it + # shadows is what every other part of this layer reads. + annotated = declared_class_vars(cls) + shadowed = [ + name + for name, annotation in own_annotations(cls).items() + if name in annotated and annotated[name] is not cls and not is_class_var(annotation) + ] + if len(shadowed) == 0: + return None + return ( + f"{cls.__name__} declares {', '.join(shadowed)} as a field, " + "shadowing a class variable of the same name" + ) + + +def _owed_class_variables_are_declared(cls: type[MetadataEntity]) -> str | None: + # A class variable annotated with no value anywhere in the ancestry + # is one the concrete entity owes: `identifier` for all of them, + # `kind` for a codec, `bounds` for an integer type. Derived rather + # than listed, so adding one to a family cannot forget to require it. + missing = [name for name in declared_class_vars(cls) if not hasattr(cls, name)] + if len(missing) == 0: + return None + return f"{cls.__name__} does not declare {', '.join(sorted(missing))}" + + +def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: + """Each member's declared default, or `_MISSING_DEFAULT`. + + `@dataclass` has not run yet -- `__init_subclass__` runs first -- so + a member declared with `field(...)` is still a `Field` here and its + default has to be unwrapped. + """ + defaulted: dict[str, object] = {} + for key in cls.member_types: + declared: object = getattr(cls, key, _MISSING_DEFAULT) + if type(declared) is Field: + spec = cast("Field[object]", declared) + declared = ( + _MISSING_DEFAULT + if spec.default is MISSING and spec.default_factory is MISSING + else spec.default + ) + defaulted[key] = declared + return defaulted + + +def _optional_members_default_to_unset(cls: type[MetadataEntity]) -> str | None: + # Or `configuration` emits the member for every instance, so the + # bare-name spelling becomes unreachable and a document gains a + # member it never wrote. + defaulted = _declared_defaults(cls) + invented = [ + key + for key, (required, _) in cls.member_types.items() + if not required and defaulted[key] is not UNSET + ] + if len(invented) == 0: + return None + return ( + f"{cls.__name__} gives the optional member(s) " + f"{', '.join(invented)} a default other than UNSET" + ) + + +def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: + # A required member with a default is an entity that can be built + # without it -- and then serializes a document nobody wrote. A + # conventional starting point is a `create_default` classmethod, + # named so that asking for one is deliberate. + defaulted = _declared_defaults(cls) + presumed = [ + key + for key, (required, _) in cls.member_types.items() + if required and defaulted[key] is not _MISSING_DEFAULT + ] + if len(presumed) == 0: + return None + return ( + f"{cls.__name__} gives the required member(s) " + f"{', '.join(presumed)} a default; required members have none" + ) + + +_INVARIANTS: Final[tuple[Callable[[type[MetadataEntity]], str | None], ...]] = ( + _final_methods_are_not_overridden, + _named_json_type_matches_what_is_written, + _nested_kinds_have_a_point, + _fields_do_not_shadow_class_variables, + _owed_class_variables_are_declared, + _optional_members_default_to_unset, + _required_members_have_no_default, +) +"""What a compiled entity must satisfy, asked in this order at class creation.""" + + @dataclass(frozen=True) class MetadataEntity(Generic[JSONT_co]): """One named entity, coerced from its metadata. @@ -598,12 +830,14 @@ class MetadataEntity(Generic[JSONT_co]): """ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: - """Refuse a subclass that is not an entity this layer can use. + """Compile the entity from its fields, and refuse one this layer cannot use. - Every check here has the same shape: something that type-checks - cleanly and then goes wrong later, somewhere that will not name - this class. An import-time error in the extension's own module is - the one place the author is looking. + `_compile_entity` derives the tables the layer reads -- the + members and their checks, the nested fields, the value rules -- + and then every invariant in `_INVARIANTS` is asked. Each names + something that type-checks cleanly and then goes wrong later, + somewhere that will not name this class; an import-time error in + the extension's own module is the one place the author is looking. `base=True` for a class that exists to add a class variable rather than to be an entity -- `CodecEntity`, `IntegerDataType`. @@ -611,189 +845,11 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: super().__init_subclass__(**kwargs) if base: return - if "problems" in cls.__dict__: - # Value rules are `value_problems`, a static routine over the - # members. An override named `problems` is a rule that would - # never run, and nothing else would say so. - msg = ( - f"{cls.__name__} defines `problems`; a value rule is a bound on the " - "field, a `@validates` rule about one member, or `value_problems` " - "over the members together -- none of them takes an entity" - ) - raise TypeError(msg) - if "prepare" in cls.__dict__: - # A member that is an entity is read from its annotation, so - # an override named `prepare` is resolution that would never - # run, and nothing else would say so. - msg = ( - f"{cls.__name__} defines `prepare`; a member that is an entity is read " - "from its field annotation, and nothing calls `prepare`" - ) - raise TypeError(msg) - if "canonical" in cls.__dict__: - # The walk into contained entities is read off the annotations - # and must not be lost under an override; the entity's own - # rewrite has a hook of its own. - msg = ( - f"{cls.__name__} overrides `canonical`, which is the walk into contained " - "entities; put the entity's own rewrite in `simplified`, which " - "`canonical` calls after the walk" - ) - raise TypeError(msg) - if "__post_init__" in cls.__dict__: - # `coerce` builds through `unchecked`, which bypasses - # `__init__` and so never reaches `__post_init__`. Rules put - # there would hold for a hand-built entity and be silently - # absent for every entity read from a document -- the one - # direction that matters. - msg = ( - f"{cls.__name__} defines `__post_init__`, which `unchecked` does " - "not reach; value rules belong in `value_problems`" - ) - raise TypeError(msg) - if "configuration_required" in vars(cls): - msg = ( - f"{cls.__name__} declares `configuration_required`, which follows " - "from whether any member is required" - ) - raise TypeError(msg) - # Before every guard below, because they read the table. - cls.member_types, unread = derive_member_types(cls) - if len(unread) != 0: - msg = ( - f"{cls.__name__}: no check can be read off the annotation of " - f"{', '.join(sorted(unread))}; teach the compiler that shape with `register_check`" - ) - raise TypeError(msg) - cls.configuration_required = any(required for required, _ in cls.member_types.values()) - json_type = json_type_of(cls) - if json_type is not ZarrV3MetadataFieldJSON: - # The named type is a promise about what `to_json` writes, and - # its shape follows from the members: a bare name only when no - # member is required and the entity must be understood, an - # object whenever there is a member to write or the flag to. - admits_bare, admits_object = _json_shape(json_type) - writes_bare = not cls.configuration_required and cls.must_understand - writes_object = len(cls.member_types) != 0 or not cls.must_understand - if admits_bare != writes_bare or admits_object != writes_object: - msg = ( - f"{cls.__name__} names {json_type!r} as its JSON type, which " - f"{'admits' if admits_bare else 'lacks'} a bare name and " - f"{'admits' if admits_object else 'lacks'} an object, but the entity " - f"{'writes' if writes_bare else 'never writes'} a bare name and " - f"{'writes' if writes_object else 'never writes'} an object" - ) - raise TypeError(msg) - hints = field_hints(cls) - cls.nested_members = { - name: annotation for name, annotation in hints.items() if contains_entity(annotation) - } - cls.value_checks = { - name: check - for name, annotation in hints.items() - if (check := value_check_for(annotation)) is not None - } - attributes: dict[str, object] = {} - for ancestor in reversed(cls.__mro__): - attributes.update(vars(ancestor)) - rules: dict[str, list[MemberRule]] = {} - for attribute in attributes.values(): - function = attribute.__func__ if isinstance(attribute, staticmethod) else attribute - # Only a function can carry the mark; a table-valued class - # attribute is not even hashable. - if not callable(function): - continue - for member in rule_members(function): - if member not in hints: - msg = f"{cls.__name__}: `@validates({member!r})` names no field of the entity" - raise TypeError(msg) - rules.setdefault(member, []).append(cast("MemberRule", function)) - cls.member_rules = {member: tuple(found) for member, found in rules.items()} - unplaced = sorted( - name - for name, annotation in cls.nested_members.items() - if any(kind.extension_point is None for kind in _entity_kinds(annotation)) - ) - if len(unplaced) != 0: - # `MetadataEntity` itself is registered at no single point, so - # a field typed as one could not be resolved through a scope. - msg = ( - f"{cls.__name__}: the entity kind of {', '.join(unplaced)} has no " - "`extension_point`; annotate it with `CodecEntity`, `DataTypeEntity` " - "or `ChunkGridEntity`" - ) - raise TypeError(msg) - annotated = declared_class_vars(cls) - shadowed = [ - name - for name, annotation in own_annotations(cls).items() - if name in annotated and annotated[name] is not cls and not is_class_var(annotation) - ] - if len(shadowed) != 0: - # A field of that name would go into `member_types`, into the - # configuration, and into the JSON -- while the class variable - # it shadows is what every other part of this layer reads. - msg = ( - f"{cls.__name__} declares {', '.join(shadowed)} as a field, " - "shadowing a class variable of the same name" - ) - raise TypeError(msg) - # A class variable annotated with no value anywhere in the - # ancestry is one the concrete entity owes: `identifier` for all - # of them, `kind` for a codec, `bounds` for an integer type. - # Derived rather than listed, so adding one to a family cannot - # forget to require it. - missing = [name for name in annotated if not hasattr(cls, name)] - if len(missing) != 0: - msg = f"{cls.__name__} does not declare {', '.join(sorted(missing))}" - raise TypeError(msg) - # A member's default decides whether the entity can exist without - # it, so the two kinds have opposite rules. `@dataclass` has not - # run yet, so a member declared with `field(...)` is still a - # `Field` here and its default has to be unwrapped. - defaulted: dict[str, object] = {} - for key in cls.member_types: - declared: object = getattr(cls, key, _MISSING_DEFAULT) - if type(declared) is Field: - # `field(...)`, so the default is inside it rather than - # being the attribute. `@dataclass` has not unwrapped it - # yet -- this hook runs first. - spec = cast("Field[object]", declared) - declared = ( - _MISSING_DEFAULT - if spec.default is MISSING and spec.default_factory is MISSING - else spec.default - ) - defaulted[key] = declared - # An optional member defaults to UNSET or `configuration` emits it - # for every instance, so the bare-name spelling becomes - # unreachable and a document gains a member it never wrote. - invented = [ - key - for key, (required, _) in cls.member_types.items() - if not required and defaulted[key] is not UNSET - ] - if len(invented) != 0: - msg = ( - f"{cls.__name__} gives the optional member(s) " - f"{', '.join(invented)} a default other than UNSET" - ) - raise TypeError(msg) - # A required member with a default is an entity that can be built - # without it -- and then serializes a document nobody wrote. A - # conventional starting point is a `create_default` classmethod, - # named so that asking for one is deliberate. - presumed = [ - key - for key, (required, _) in cls.member_types.items() - if required and defaulted[key] is not _MISSING_DEFAULT - ] - if len(presumed) != 0: - msg = ( - f"{cls.__name__} gives the required member(s) " - f"{', '.join(presumed)} a default; required members have none" - ) - raise TypeError(msg) + _compile_entity(cls) + for invariant in _INVARIANTS: + message = invariant(cls) + if message is not None: + raise TypeError(message) @classmethod def accepts(cls, name: str) -> bool: @@ -844,6 +900,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # Already asked, so do not ask again on the way in. return cls.unchecked(**members), found + @final def canonical(self) -> Self: """This entity in the simplest form that means the same thing. @@ -931,6 +988,7 @@ def configuration(self) -> dict[str, object]: Locations are relative to the entity's `configuration`. """ + @final def __post_init__(self) -> None: """Refuse to exist with values the spec disallows. diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 9c02aecf3e..a521d6b03f 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast import pytest @@ -262,20 +262,6 @@ def test_a_reader_can_choose_its_own_scope() -> None: assert in_scope.acceleration == 4 -def test_error_value_rules_must_be_value_problems() -> None: - # `problems` was the old name and takes an entity; an override using - # it would never run, and nothing else would notice. - with pytest.raises(TypeError, match="none of them takes an entity"): - - @dataclass(frozen=True) - class Stale(CodecEntity): # pyright: ignore[reportUnusedClass] - identifier: ClassVar[str] = "acme.stale" - kind: ClassVar[CodecKind] = "bytes_bytes" - - def problems(self) -> tuple[ValidationProblem, ...]: - return () - - def test_error_an_entity_may_not_validate_in_post_init() -> None: # `coerce` builds through `unchecked`, which never reaches # `__post_init__`, so a rule there holds for a hand-built entity and @@ -287,7 +273,7 @@ class Eager(CodecEntity): # pyright: ignore[reportUnusedClass] identifier: ClassVar[str] = "acme.eager" kind: ClassVar[CodecKind] = "bytes_bytes" - def __post_init__(self) -> None: + def __post_init__(self) -> None: # pyright: ignore[reportIncompatibleMethodOverride] raise AssertionError @@ -379,18 +365,20 @@ class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] kind: ClassVar[CodecKind] = "bytes_bytes" -def test_error_a_bare_name_rule_may_not_be_restated() -> None: - # Whether the bare spelling is legal follows from whether any member - # is required, which the fields already say. - with pytest.raises(TypeError, match="declares `configuration_required`"): - - @dataclass(frozen=True) - class Opinionated(CodecEntity): # pyright: ignore[reportUnusedClass] - acceleration: int | UNSET = UNSET - - identifier: ClassVar[str] = "acme.opinionated" - kind: ClassVar[CodecKind] = "bytes_bytes" - configuration_required: ClassVar[bool] = True +@pytest.mark.parametrize( + "name", + ["member_types", "configuration_required", "nested_members", "value_checks", "member_rules"], +) +def test_error_a_derived_class_variable_may_not_be_declared(name: str) -> None: + # Each is read off the fields at class creation, and a declaration + # would be silently overwritten by that reading. Built with `type`, + # since a class body cannot spell a name from a parameter. + with pytest.raises(TypeError, match=f"declares {name}, which is derived from the fields"): + type( + "Opinionated", + (CodecEntity,), + {"identifier": "acme.opinionated", "kind": "bytes_bytes", name: {}}, + ) # A third-party codec that contains another codec: the case that used to @@ -464,58 +452,6 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non assert inner.typesize is UNSET -def test_error_an_entity_may_not_define_prepare() -> None: - # A member that is an entity is read from its annotation; an override - # named `prepare` is resolution that would never run. - with pytest.raises(TypeError, match="nothing calls `prepare`"): - - @dataclass(frozen=True) - class Preparer(CodecEntity): # pyright: ignore[reportUnusedClass] - identifier: ClassVar[str] = "acme.preparer" - kind: ClassVar[CodecKind] = "bytes_bytes" - - @classmethod - def prepare(cls, members: object, context: object) -> object: - return members - - -def test_error_an_entity_may_not_override_canonical() -> None: - # `canonical` is the walk into contained entities, read off the - # annotations; an override could lose it. The entity's own rewrite - # goes in `simplified`. - with pytest.raises(TypeError, match="put the entity's own rewrite in `simplified`"): - - @dataclass(frozen=True) - class Rewriter(CodecEntity): # pyright: ignore[reportUnusedClass] - identifier: ClassVar[str] = "acme.rewriter" - kind: ClassVar[CodecKind] = "bytes_bytes" - - def canonical(self) -> Self: - return self - - -def test_simplified_composes_with_the_walk_into_contained_entities() -> None: - # An entity that contains an entity and rewrites its own members gets - # both from `canonical` -- the contained blosc loses the `typesize` - # that `noshuffle` ignores, and the frame of 0 that means "unframed" - # is dropped -- with nothing to call `super()` for. - @dataclass(frozen=True) - class AcmeFramedCodec(CodecEntity): - inner: CodecEntity | Opaque - frame: int | UNSET = UNSET - - identifier: ClassVar[str] = "acme.framed" - kind: ClassVar[CodecKind] = "bytes_bytes" - - def simplified(self) -> Self: - return self if self.frame != 0 else replace(self, frame=UNSET) - - blosc = BloscCodec(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) - framed = AcmeFramedCodec(inner=blosc, frame=0) - assert framed.canonical() == AcmeFramedCodec(inner=replace(blosc, typesize=UNSET)) - assert framed.inner is blosc # a transformation, not a mutation - - def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: # `MetadataEntity` is registered at no single point, so a field typed # as one could not be resolved through any scope. From 4a136055f3337e75c9a6fb165a2f9909bff90af0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 12:35:55 +0200 Subject: [PATCH 072/107] test(zarr-metadata): restore the two canonical tests the tombstone cut removed The guard test for overriding `canonical` and the test that `simplified` composes with the walk into contained entities sat between the two tests deleted with the `prepare` guard, and went with them. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../tests/v3/test_extension_api.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index a521d6b03f..7cc20dcedc 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast import pytest @@ -452,6 +452,43 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non assert inner.typesize is UNSET +def test_error_an_entity_may_not_override_canonical() -> None: + # `canonical` is the walk into contained entities, read off the + # annotations; an override could lose it. The entity's own rewrite + # goes in `simplified`. + with pytest.raises(TypeError, match="put the entity's own rewrite in `simplified`"): + + @dataclass(frozen=True) + class Rewriter(CodecEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.rewriter" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def canonical(self) -> Self: # pyright: ignore[reportIncompatibleMethodOverride] + return self + + +def test_simplified_composes_with_the_walk_into_contained_entities() -> None: + # An entity that contains an entity and rewrites its own members gets + # both from `canonical` -- the contained blosc loses the `typesize` + # that `noshuffle` ignores, and the frame of 0 that means "unframed" + # is dropped -- with nothing to call `super()` for. + @dataclass(frozen=True) + class AcmeFramedCodec(CodecEntity): + inner: CodecEntity | Opaque + frame: int | UNSET = UNSET + + identifier: ClassVar[str] = "acme.framed" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def simplified(self) -> Self: + return self if self.frame != 0 else replace(self, frame=UNSET) + + blosc = BloscCodec(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) + framed = AcmeFramedCodec(inner=blosc, frame=0) + assert framed.canonical() == AcmeFramedCodec(inner=replace(blosc, typesize=UNSET)) + assert framed.inner is blosc # a transformation, not a mutation + + def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: # `MetadataEntity` is registered at no single point, so a field typed # as one could not be resolved through any scope. From aee137c498dd0dc89cc8ab0af86cfbd28b782c4b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 13:21:39 +0200 Subject: [PATCH 073/107] refactor(zarr-metadata): refinements are __post_init__; the compiler reads JSON shapes only The type check over an entity's fields stays: bounded, because every field is one of the shapes JSON takes. Everything finer -- a bound, a rule about one member, members read together -- is now the entity's own `__post_init__`, in plain code, collecting every problem and raising `MetadataValidationError` once; `coerce` builds the same way, catches the same error, and reports the problems located under the configuration. The space of refinements is too wide to capture statically, and seeing them whole per entity is what will show where a shared form is worth having. Gone with that: the `Annotated` bound vocabulary and its compiler, the `@validates` mark and its side table, `value_problems`, `_judge_values`, `unchecked`, the `@final` on `__post_init__`, and the open shape registry -- `register_check` handled arbitrary types, and no field is one. The one shape the compiler cannot recognise by itself, a nested metadata field, comes through `nested_field`, which `_entity` sets. 40k-document differential: zero problems lost or gained. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.10.md | 20 +- .../zarr-metadata/changes/4379.feature.7.md | 44 +- packages/zarr-metadata/changes/4379.misc.2.md | 5 +- .../src/zarr_metadata/v3/_compile.py | 442 +++--------------- .../src/zarr_metadata/v3/_entity.py | 263 ++--------- .../v3/chunk_grid/rectilinear.py | 40 +- .../zarr_metadata/v3/chunk_grid/regular.py | 21 +- .../src/zarr_metadata/v3/codec/blosc.py | 60 ++- .../src/zarr_metadata/v3/codec/gzip.py | 15 +- .../zarr_metadata/v3/codec/scale_offset.py | 33 +- .../v3/codec/sharding_indexed.py | 21 +- .../src/zarr_metadata/v3/codec/transpose.py | 18 +- .../src/zarr_metadata/v3/codec/zstd.py | 17 +- .../v3/data_type/numpy_datetime64.py | 18 +- .../v3/data_type/numpy_timedelta64.py | 18 +- .../src/zarr_metadata/v3/data_type/raw.py | 31 +- .../src/zarr_metadata/v3/data_type/struct.py | 30 +- .../src/zarr_metadata/v3/entity.py | 63 +-- .../zarr-metadata/tests/test_public_api.py | 7 - .../zarr-metadata/tests/v3/test_entities.py | 19 - .../tests/v3/test_extension_api.py | 141 ++---- 21 files changed, 373 insertions(+), 953 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index 84ad4a2597..c88e61a831 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -5,15 +5,11 @@ such thing as an invalid entity to hold. Code that wants unvalidated metadata already has somewhere to put it: `Opaque`, which carries the JSON verbatim. -Three pieces, as the structure requires. `value_problems` is a static -routine over the members rather than a method on an entity, because -judging values does not need one -- and needing one would mean an invalid -one had been built. It is annotated with the members it judges -(`Unpack[...]`), so the entity's dataclass fields, its configuration -TypedDict, its member table and its value routine are four spellings of -one set, and a test holds them to it. `unchecked` builds without asking, -for a caller that has already asked. The dataclass constructor asks, then -builds. +One piece: the entity's `__post_init__`, in plain code, which collects +every value the spec disallows and raises `MetadataValidationError` once. +The constructor asks, then builds; `coerce` builds the same way, catches +the same error, and reports its problems in the document instead of +raising them. `must_understand` moves to a class variable and out of the configuration entirely. It is a property of the *kind* of metadata -- a codec is @@ -44,7 +40,5 @@ Registering an entity is checked at class creation, because every way of getting it wrong type-checks cleanly and then fails somewhere that will not name the class: a class variable the entity owes and did not declare (read off the annotations, so a family adding one cannot forget to -require it), a field shadowing one, a value rule in `__post_init__` -- -which `unchecked` never reaches, so it would hold for a hand-built entity -and be silently absent for every entity read from a document -- and a -member whose default contradicts whether the spec requires it. +require it), a field shadowing one, and a member whose default +contradicts whether the spec requires it. diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index ab4b579324..9619572eab 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -75,36 +75,16 @@ dimension's run-length encoding, a `typesize` that `noshuffle` ignores override of `canonical` itself is refused at class creation, so the walk cannot be lost and there is no `super()` to remember. -A bound on a value is written on the field, in the `annotated_types` -vocabulary -- `level: Annotated[int, Interval(ge=0, le=9)]`, -`chunk_shape: tuple[Annotated[int, Ge(1)], ...]` -- and judged at -whatever depth the annotation puts it, so a bound on an element type -locates its finding at the element. Seven value routines that stated -nothing but a bound are gone: gzip's and zstd's levels, the two time -types' scale factors, and the positivity of every chunk extent in the -regular grid, the sharding codec and the rectilinear grid, run-length -counts included. `value_problems` remains for a rule that is not a -bound. The bounds run with it, after the type checks, on the reading -path and in the constructor alike, through one routine. +Everything finer than a type -- a bound, a rule about one member, a +rule that reads two members together -- is the entity's own +`__post_init__`, in plain code: it collects every problem it finds, +located relative to the configuration, and raises `MetadataValidationError` +once, so `BloscCodec(clevel=99)` raises; `coerce` catches the same +error and reports the problems in the document instead. The space of +refinements is too wide to capture statically, and reading them in one +place per entity is what will show where a shared form is worth having. -A rule about one member that is not a bound is a `@validates` rule: a -staticmethod that takes the member's value, runs only when the member -is present and has the type it declared, and reports relative to the -member. The decorator hands back the function it was given and records -it in a side table, so the declared signature survives for the type -checker. Transpose's permutation, `scale_offset`'s two not-null rules -(one rule, two members) and `struct`'s field rules are written that way. -`value_problems` remains for a rule that reads two members together, -which after this is exactly one: blosc's `typesize` against its -`shuffle`. Three places a value rule can live, by what it is about: a -bound on the field, a member's rule under `@validates`, the members -together in `value_problems`. - -The set of annotation shapes the compiler reads is open. Each built-in -shape is a registration -- a predicate over the annotation and what to -compile it to -- consulted in order, and `register_check` adds one from -outside, ahead of the built-ins, so a package with a field type this -package does not read (a `str` subclass, say) teaches the compiler once -rather than declaring a check on every entity that uses it. The same -door the built-in shapes came through; cattrs' `register_structure_hook_func` -is the pattern. +The set of annotation shapes the compiler reads is closed: the shapes +JSON takes, and no others. A field annotation outside them is refused +at class creation, and the field is written as one of them instead, with +any finer rule in `__post_init__`. diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 9a8e767a5d..b8d0835034 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -42,9 +42,8 @@ third-party field typed with either. Class creation is two steps with one owner each: `_compile_entity` derives the tables the layer reads from the fields, and `_INVARIANTS`, a declared tuple of named functions, asks each invariant of the -compiled class in order. `canonical` and `__post_init__` are `@final`, -so an override is refused by pyright in the author's editor as well as -at class creation; the guards that refused `problems` and `prepare` -- +compiled class in order. `canonical` is `@final`, so an override is +refused by pyright in the author's editor as well as at class creation; the guards that refused `problems` and `prepare` -- names from earlier drafts of this branch, never released -- are gone, and declaring any derived class variable is refused, where before only `configuration_required` was. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py index 12f90ef911..9aef99bd41 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py @@ -1,19 +1,17 @@ -"""What a field annotation says, read off it: the type, the bounds, the rules. +"""What a field annotation says, read off it: the type. An entity's dataclass fields are its schema, and this module is the compiler over them. `check_for` turns an annotation into its type check -- a scalar, a `Literal`, arrays homogeneous or fixed, unions, a nested -object described by a TypedDict or a record dataclass -- through a -registry of shapes that `register_check` keeps open, so a shape this -module does not know can be taught to it from outside. The bound -vocabulary (`Ge`, `Le`, `Interval`, ...) rides in `Annotated` and -`value_check_for` compiles it, at whatever depth it sits. `validates` -marks a rule about one member. `derive_member_types` is what an entity -reads its member table off. - -Nothing here knows what an entity is. A nested metadata field is a shape -like any other, registered by `_entity` through the same door, with the -JSON shape and the description the compiler needs of it. +object described by a TypedDict or a record dataclass, an object of +undeclared keys, a `NewType` as the type it names: the shapes JSON takes +and no others, which is what keeps it small. `derive_member_types` is +what an entity reads its member table off. Anything finer than a type +-- a bound, a rule about a member, members read together -- is the +entity's own `__post_init__`, in plain code. + +Nothing here knows what an entity is. A nested metadata field is the one +shape recognised through `nested_field`, which `_entity` sets. """ from __future__ import annotations @@ -25,7 +23,7 @@ import sys import types from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, is_dataclass +from dataclasses import is_dataclass from typing import ( TYPE_CHECKING, Annotated, @@ -35,8 +33,6 @@ NewType, NotRequired, Required, - TypeAlias, - TypeVar, Union, cast, get_args, @@ -53,6 +49,7 @@ is_int, is_integer, is_json_value, + is_metadata_field, is_str, object_of, one_of, @@ -86,52 +83,6 @@ def __repr__(self) -> str: """ -@dataclass(frozen=True, slots=True) -class Ge: - """`Annotated[int, Ge(1)]`: the value is at least `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Gt: - """`Annotated[int, Gt(0)]`: the value is more than `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Le: - """`Annotated[int, Le(9)]`: the value is at most `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Lt: - """`Annotated[int, Lt(10)]`: the value is less than `bound`.""" - - bound: int | float - - -@dataclass(frozen=True, slots=True) -class Interval: - """`Annotated[int, Interval(ge=0, le=9)]`: the value lies within these bounds. - - These five are the `annotated_types` vocabulary -- what pydantic reads - and msgspec's `Meta` mirrors -- so a reader recognises them. Defined - here rather than imported, so the package keeps its one dependency. - A bound is a value rule: it runs only once the member has the type it - declared, at whatever depth the annotation puts it, so a bound on an - array's element type judges each element at its own position. - """ - - ge: int | float | None = None - gt: int | float | None = None - le: int | float | None = None - lt: int | float | None = None - - def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: """An annotation's type, and the metadata `Annotated` wrapped it in. @@ -204,12 +155,25 @@ def is_optional(annotation: object) -> bool: return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) +def _no_nested_field(annotation: object) -> bool: + return False + + +nested_field: Callable[[object], bool] = _no_nested_field +"""Whether an annotation is a nested metadata field: an entity type, or a union of those with `Opaque`. + +The one shape the compiler cannot recognise by itself, because which +classes are entities is `_entity`'s to say; it sets this once at import. +Consulted ahead of every other shape, since an entity is a dataclass too +and must not be walked as a record. +""" + + def describe(annotation: object) -> str: """The annotation as a message would name it: "an integer", "an object".""" inner, _ = strip_annotation(annotation) - for registration in _CHECK_COMPILERS: - if registration.description is not None and registration.predicate(inner): - return registration.description + if nested_field(inner): + return "a metadata field" if inner is int: return "an integer" if inner is bool: @@ -246,9 +210,8 @@ def shape_of(annotation: object) -> str | None: None means any shape -- a JSON value, or a union that mixes them. """ inner, _ = strip_annotation(annotation) - for registration in _CHECK_COMPILERS: - if registration.shape is not None and registration.predicate(inner): - return registration.shape + if nested_field(inner): + return "field" if inner is int: return "int" if inner is bool: @@ -364,94 +327,6 @@ def _members_of(annotations: Mapping[str, object]) -> dict[str, tuple[bool, Type return members -CheckCompiler: TypeAlias = "Callable[[object], TypeCheck | None]" -"""Turns one annotation into its type check -- or None, to decline it after all.""" - - -@dataclass(frozen=True, slots=True) -class Registration: - """One shape `check_for` reads: how to recognise it, and what to make of it. - - `shape` and `description` are for a shape the compiler's own logic - does not know -- a nested metadata field, say -- so that choosing a - union branch and naming the shape in a message work for it too. - """ - - predicate: Callable[[object], bool] - compile: CheckCompiler - shape: str | None = None - description: str | None = None - - -_CHECK_COMPILERS: Final[list[Registration]] = [] -"""The shapes `check_for` reads, consulted front to back. - -The built-in shapes are appended below in the order they must be tried, -and `register_check` puts a registration in front of all of them, so the -newest one wins. `_entity` registers the nested metadata field this way, -ahead of the record shape -- an entity is a dataclass too. -""" - - -def register_check( - predicate: Callable[[object], bool], - compile: CheckCompiler, - *, - shape: str | None = None, - description: str | None = None, -) -> None: - """Teach `check_for` an annotation shape it does not read. - - class Hex(str): ... - register_check(lambda annotation: annotation is Hex, lambda annotation: is_hex) - - `predicate` sees the annotation with `Annotated`, `NotRequired` and - `ReadOnly` peeled; `compile` returns the check for it, calling - `check_for` itself for any shape inside. A registration is consulted - before every built-in one, so a package can also replace how a - built-in shape is judged. The same door the built-ins came through, - which is what makes the set of shapes open rather than this module's. - - `shape` names the JSON shape the annotation admits, for choosing the - branch of a union a value fits (`"int"`, `"str"`, `"tuple"`, - `"mapping"`, or `"field"` for a bare name or object), and - `description` is how a message names it; both are needed only for a - shape the compiler's own logic does not recognise. - """ - _CHECK_COMPILERS.insert(0, Registration(predicate, compile, shape, description)) - - -def _builtin(predicate: Callable[[object], bool]) -> Callable[[CheckCompiler], CheckCompiler]: - """Register a built-in shape, in the order written.""" - - def append(compile: CheckCompiler) -> CheckCompiler: - _CHECK_COMPILERS.append(Registration(predicate, compile)) - return compile - - return append - - -@_builtin(lambda inner: inner is int) -def _compile_int(inner: object) -> TypeCheck | None: - return is_int - - -@_builtin(lambda inner: inner is bool) -def _compile_bool(inner: object) -> TypeCheck | None: - return is_bool - - -@_builtin(lambda inner: inner is str) -def _compile_str(inner: object) -> TypeCheck | None: - return is_str - - -@_builtin(lambda inner: inner is JSONValue) -def _compile_json_value(inner: object) -> TypeCheck | None: - return is_json_value - - -@_builtin(lambda inner: get_origin(inner) is Literal) def _compile_literal(inner: object) -> TypeCheck | None: # Sorted, because the order `get_args` reports is not the order the # `Literal` was written in: two `Literal`s over the same values @@ -462,7 +337,6 @@ def _compile_literal(inner: object) -> TypeCheck | None: return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) -@_builtin(is_union) def _compile_union(inner: object) -> TypeCheck | None: branches = [arg for arg in get_args(inner) if arg is not UNSET] if len(branches) == 1: @@ -475,7 +349,6 @@ def _compile_union(inner: object) -> TypeCheck | None: ) -@_builtin(lambda inner: get_origin(inner) is tuple) def _compile_tuple(inner: object) -> TypeCheck | None: arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: @@ -487,13 +360,11 @@ def _compile_tuple(inner: object) -> TypeCheck | None: return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) -@_builtin(is_typeddict) def _compile_typeddict(inner: object) -> TypeCheck | None: members = _members_of(get_type_hints(inner, include_extras=True)) return None if members is None else mapping_of(members) -@_builtin(lambda inner: isinstance(inner, type) and is_dataclass(inner)) def _compile_record(inner: object) -> TypeCheck | None: if not isinstance(inner, type): # pragma: no cover - the predicate says it is return None @@ -501,7 +372,6 @@ def _compile_record(inner: object) -> TypeCheck | None: return None if members is None else mapping_of(members) -@_builtin(lambda inner: get_origin(inner) in (Mapping, dict)) def _compile_mapping(inner: object) -> TypeCheck | None: # An object of undeclared keys: `Mapping[str, V]`, every value a `V`. arguments = get_args(inner) @@ -511,7 +381,6 @@ def _compile_mapping(inner: object) -> TypeCheck | None: return None if value is None else object_of(value) -@_builtin(lambda inner: isinstance(inner, NewType)) def _compile_new_type(inner: object) -> TypeCheck | None: # A `NewType` is its supertype to a document; the distinction is the # code's, for a value it has vouched for. @@ -521,24 +390,46 @@ def _compile_new_type(inner: object) -> TypeCheck | None: def check_for(annotation: object) -> TypeCheck | None: """The type check a field annotation implies, or None if it implies none. - A small compiler over the shapes this package's metadata takes: the - JSON scalars, a `Literal` of names, arrays homogeneous or fixed, - unions of those, a nested object described by a TypedDict or a - record dataclass, an object of undeclared keys as `Mapping[str, V]`, - a `NewType` as the type it names, and a nested metadata field -- an - entity type, with or without `Opaque`. `UNSET` in a union says the - member may be absent, which is the other half of a table entry and - is read separately by `is_optional`. - - Open: each shape is a registration in `_CHECK_COMPILERS`, and - `register_check` adds one from outside. None for an annotation no - registration claims, which the entity then declares a check for by - hand. + A small compiler over the shapes JSON takes, and no others: the + scalars, a `Literal` of names, arrays homogeneous or fixed, unions of + those, a nested object described by a TypedDict or a record dataclass, + an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the + type it names, and a nested metadata field -- an entity type, with or + without `Opaque`, which `nested_field` recognises. `UNSET` in a union + says the member may be absent, which is the other half of a table + entry and is read separately by `is_optional`. + + Closed: an annotation outside these implies no check, and an entity + declaring one is refused at class creation. The field is written as + one of these shapes instead, with any finer rule in `__post_init__`. """ inner, _ = strip_annotation(annotation) - for registration in _CHECK_COMPILERS: - if registration.predicate(inner): - return registration.compile(inner) + if nested_field(inner): + return is_metadata_field + if inner is int: + return is_int + if inner is bool: + return is_bool + if inner is str: + return is_str + if inner is JSONValue: + return is_json_value + if get_origin(inner) is Literal: + return _compile_literal(inner) + if is_union(inner): + return _compile_union(inner) + if get_origin(inner) is tuple: + return _compile_tuple(inner) + if is_typeddict(inner): + return _compile_typeddict(inner) + if get_origin(inner) in (Mapping, dict): + return _compile_mapping(inner) + if isinstance(inner, NewType): + return _compile_new_type(inner) + # Last, because `is_dataclass` narrows what pyright knows of `inner` + # for every line after it. + if isinstance(inner, type) and is_dataclass(inner): + return _compile_record(inner) return None @@ -548,8 +439,8 @@ def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], l Every field is a configuration member unless `FROM_NAME` says it is carried by the envelope. Requiredness is whether the type admits `UNSET`; the check is whatever `check_for` reads off the type. Also - returned: the fields no check could be read for, which the entity - must declare by hand. + returned: the fields no check could be read for, which class + creation refuses. """ derived: dict[str, tuple[bool, TypeCheck]] = {} unread: list[str] = [] @@ -565,187 +456,6 @@ def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], l return derived, unread -MemberRule: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" -"""A rule about one member: takes its value, reports relative to it.""" - - -_RULE_MEMBERS: Final[dict[object, tuple[str, ...]]] = {} -"""Which members each `@validates` rule is about, keyed by the function. - -A side table rather than an attribute on the function, so the decorator -hands back exactly what it was given -- the declared signature survives, -and the type checker keeps checking the body and its callers. -""" - - -_Rule = TypeVar("_Rule", bound="Callable[..., tuple[ValidationProblem, ...]]") - - -def rule_members(function: object) -> tuple[str, ...]: - """The members a function was marked as a rule about, if any.""" - return _RULE_MEMBERS.get(function, ()) - - -def validates(*members: str) -> Callable[[_Rule], _Rule]: - """Mark a static rule as being about one member, or several alike. - - @staticmethod - @validates("order") - def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: - ... - - The rule receives the member's value, already of the type the field - declares, and only when the member is present; it reports relative - to the member, so a problem with an empty location is about the - member itself. Naming several members applies the one rule to each. - A rule that reads two members together is `value_problems`. - """ - - def mark(rule: _Rule) -> _Rule: - _RULE_MEMBERS[rule] = members - return rule - - return mark - - -def _bound_check(metadata: Sequence[object]) -> TypeCheck | None: - """The check the bound markers among an annotation's metadata imply, or None.""" - ge = gt = le = lt = None - for marker in metadata: - if isinstance(marker, Ge): - ge = marker.bound - elif isinstance(marker, Gt): - gt = marker.bound - elif isinstance(marker, Le): - le = marker.bound - elif isinstance(marker, Lt): - lt = marker.bound - elif isinstance(marker, Interval): - ge = marker.ge if marker.ge is not None else ge - gt = marker.gt if marker.gt is not None else gt - le = marker.le if marker.le is not None else le - lt = marker.lt if marker.lt is not None else lt - if ge is None and gt is None and le is None and lt is None: - return None - if ge is not None and le is not None and gt is None and lt is None: - expectation = f"an integer in [{ge}, {le}]" - else: - comparisons = [ - text - for bound, text in ( - (ge, f">= {ge}"), - (gt, f"> {gt}"), - (le, f"<= {le}"), - (lt, f"< {lt}"), - ) - if bound is not None - ] - expectation = "an integer " + " and ".join(comparisons) - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - # Not a number: the type check's finding, not this one's. - if isinstance(value, bool) or not isinstance(value, (int, float)): - return () - within_bounds = ( - (ge is None or value >= ge) - and (gt is None or value > gt) - and (le is None or value <= le) - and (lt is None or value < lt) - ) - if within_bounds: - return () - return problem(loc, f"expected {expectation}, got {value}", "invalid_value") - - return check - - -def value_check_for(annotation: object) -> TypeCheck | None: - """The value check an annotation's metadata implies, at any depth, or None. - - Over the shapes as the entity holds them, not as the JSON spells - them: this runs after every member has its type and every nested - entity has been read, so a record is a dataclass instance here and - an entity is skipped -- it is valid by construction. - """ - inner, metadata = strip_annotation(annotation) - own = _bound_check(metadata) - below: TypeCheck | None = None - if is_union(inner): - branches = [ - (branch, value_check_for(branch)) for branch in get_args(inner) if branch is not UNSET - ] - if any(check is not None for _, check in branches): - - def by_branch(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - for branch, check in branches: - if check is not None and has_shape(shape_of(branch), value): - return check(value, loc) - return () - - below = by_branch - elif get_origin(inner) is tuple: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - element = value_check_for(arguments[0]) - if element is not None: - each = element - - def per_element(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - entries = cast("tuple[object, ...]", value) - return tuple( - found - for position, entry in enumerate(entries) - for found in each(entry, (*loc, position)) - ) - - below = per_element - else: - positions = [value_check_for(argument) for argument in arguments] - if any(check is not None for check in positions): - - def per_position(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - entries = cast("tuple[object, ...]", value) - return tuple( - found - for position, (check, entry) in enumerate( - zip(positions, entries, strict=True) - ) - if check is not None - for found in check(entry, (*loc, position)) - ) - - below = per_position - elif isinstance(inner, type) and is_dataclass(inner) and shape_of(inner) != "field": - members = { - name: check - for name, field_annotation in field_hints(inner).items() - if (check := value_check_for(field_annotation)) is not None - } - if len(members) != 0: - - def per_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - return tuple( - found - for name, check in members.items() - if (held := getattr(value, name)) is not UNSET - for found in check(held, (*loc, name)) - ) - - below = per_field - if own is None and below is None: - return None - if below is None: - return own - if own is None: - return below - outer, inner_check = own, below - - def both(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - return (*outer(value, loc), *inner_check(value, loc)) - - return both - - def element_annotations(inner: object, count: int) -> list[object]: """The annotation of each element of a tuple type, one per element held.""" arguments = get_args(inner) @@ -785,13 +495,6 @@ def declared_class_vars(cls: type) -> dict[str, type]: __all__ = [ "FROM_NAME", - "CheckCompiler", - "Ge", - "Gt", - "Interval", - "Le", - "Lt", - "MemberRule", "any_of", "check_for", "declared_class_vars", @@ -805,11 +508,8 @@ def declared_class_vars(cls: type) -> dict[str, type]: "is_optional", "is_union", "mapping_of", + "nested_field", "own_annotations", - "register_check", - "rule_members", "shape_of", "strip_annotation", - "validates", - "value_check_for", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index defde4411f..a1cb2bf956 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -2,13 +2,14 @@ A codec, data type, chunk grid or chunk key encoding is one frozen dataclass whose fields are its schema. Everything the layer knows about -a member is read off the field annotations by `_compile`: which members -there are, which may be absent, how each is type-checked, the bounds it -must satisfy, and -- for a field typed as another entity -- that it is -read through the scope, written back as its own JSON, and put in -canonical form by recursing into it. The three places a value rule -lives, by what it is about: a bound on the field, a member's rule under -`@validates`, the members together in `value_problems`. +a member's type is read off the field annotations by `_compile`: which +members there are, which may be absent, how each is type-checked, and +-- for a field typed as another entity -- that it is read through the +scope, written back as its own JSON, and put in canonical form by +recursing into it. Everything finer than a type -- a bound, a rule about +a member, members read together -- is the entity's own `__post_init__`, +which collects every problem it finds and raises once; `coerce` reports +those instead of raising. `coerce` is the reading path: raw metadata in, the entity or the reasons it is not one out, taking a `Context` -- the entities in scope @@ -32,7 +33,7 @@ # then -- for this package and for any tool introspecting an entity. from collections.abc import Callable, Mapping, Sequence # noqa: TC003 from copy import deepcopy -from dataclasses import MISSING, Field, dataclass, fields, is_dataclass, replace +from dataclasses import MISSING, Field, dataclass, is_dataclass, replace from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -63,6 +64,7 @@ from zarr_metadata.v3._parts import ArrayParts from zarr_metadata.v3._registry import Context +from zarr_metadata.v3 import _compile from zarr_metadata.v3._checks import ( Loc, MemberTypes, @@ -83,13 +85,6 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._compile import ( FROM_NAME, - CheckCompiler, - Ge, - Gt, - Interval, - Le, - Lt, - MemberRule, declared_class_vars, derive_member_types, element_annotations, @@ -98,12 +93,8 @@ is_class_var, is_union, own_annotations, - register_check, - rule_members, shape_of, strip_annotation, - validates, - value_check_for, ) EntityT = TypeVar("EntityT", bound="MetadataEntity") @@ -260,17 +251,9 @@ def _is_nested_field(annotation: object) -> bool: return _is_entity_or_opaque([candidate for candidate in candidates if candidate is not UNSET]) -def _compile_nested_field(annotation: object) -> TypeCheck | None: - return is_metadata_field - - -# The compiler knows nothing of entities; this is where it learns that a -# field typed as one is a nested metadata field. Registered ahead of every -# built-in shape -- an entity is a dataclass too, and must not be walked as -# a record -- through the same door a third party's shape comes in by. -register_check( - _is_nested_field, _compile_nested_field, shape="field", description="a metadata field" -) +# The compiler knows nothing of entities; this is where it learns which +# annotations are nested metadata fields. +_compile.nested_field = _is_nested_field def contains_entity(annotation: object) -> bool: @@ -451,19 +434,10 @@ def canonicalize_nested(annotation: object, value: object) -> object: return value -ValueRoutine: TypeAlias = "Callable[..., tuple[ValidationProblem, ...]]" -"""An entity's value-space judgment, over the members it was given.""" - - _MISSING_DEFAULT: Final = object() """Distinguishes "declared no default" from a default that is None or UNSET.""" -def _no_value_problems(**members: object) -> tuple[ValidationProblem, ...]: - """An entity whose types admit only valid values has nothing to add.""" - return () - - @dataclass(frozen=True, slots=True) class Opaque: """A metadata field this reading did not turn into an entity. @@ -491,13 +465,7 @@ class Opaque: # have to spell it `super(Cls, self)`. CPython fixed this in 3.13, so when # that is the floor this is worth revisiting; the memory saved is small at # document scale, which is why it has not been. -_DERIVED: Final = ( - "member_types", - "configuration_required", - "nested_members", - "value_checks", - "member_rules", -) +_DERIVED: Final = ("member_types", "configuration_required", "nested_members") """The class variables `_compile_entity` derives; a declaration of one is refused.""" @@ -505,9 +473,8 @@ def _compile_entity(cls: type[MetadataEntity]) -> None: """Derive from the fields the tables the layer reads. Raises for a declaration that cannot be compiled: a derived table - declared by hand, which the derivation would silently overwrite; a - field annotation no registered shape reads; a `@validates` naming no - field. + declared by hand, which the derivation would silently overwrite, and + a field annotation outside the shapes the compiler reads. """ declared = [name for name in _DERIVED if name in vars(cls)] if len(declared) != 0: @@ -520,7 +487,8 @@ def _compile_entity(cls: type[MetadataEntity]) -> None: if len(unread) != 0: msg = ( f"{cls.__name__}: no check can be read off the annotation of " - f"{', '.join(sorted(unread))}; teach the compiler that shape with `register_check`" + f"{', '.join(sorted(unread))}; a field is one of the shapes JSON takes, " + "with any finer rule in `__post_init__`" ) raise TypeError(msg) cls.configuration_required = any(required for required, _ in cls.member_types.values()) @@ -528,34 +496,6 @@ def _compile_entity(cls: type[MetadataEntity]) -> None: cls.nested_members = { name: annotation for name, annotation in hints.items() if contains_entity(annotation) } - cls.value_checks = { - name: check - for name, annotation in hints.items() - if (check := value_check_for(annotation)) is not None - } - cls.member_rules = _member_rules(cls, hints) - - -def _member_rules( - cls: type[MetadataEntity], hints: Mapping[str, object] -) -> dict[str, tuple[MemberRule, ...]]: - """The `@validates` rules in the class and its ancestors, by the member each is about.""" - attributes: dict[str, object] = {} - for ancestor in reversed(cls.__mro__): - attributes.update(vars(ancestor)) - rules: dict[str, list[MemberRule]] = {} - for attribute in attributes.values(): - function = attribute.__func__ if isinstance(attribute, staticmethod) else attribute - # Only a function can carry the mark; a table-valued class - # attribute is not even hashable. - if not callable(function): - continue - for member in rule_members(function): - if member not in hints: - msg = f"{cls.__name__}: `@validates({member!r})` names no field of the entity" - raise TypeError(msg) - rules.setdefault(member, []).append(cast("MemberRule", function)) - return {member: tuple(found) for member, found in rules.items()} # The invariants, each a function of the compiled class returning why it @@ -567,10 +507,6 @@ def _member_rules( "which is the walk into contained entities; put the entity's own rewrite " "in `simplified`, which `canonical` calls after the walk" ), - "__post_init__": ( - "which `unchecked` does not reach, and `coerce` builds through `unchecked`; " - "value rules belong in `value_problems`" - ), } """What to do instead, for each method the base marks `@final`.""" @@ -740,12 +676,14 @@ class MetadataEntity(Generic[JSONT_co]): mapping instead: `MappingProxyType` is unhashable too, and anything else stops `json.dumps` from serializing what `to_json` returns. - A subclass writes its fields, and a rule where the spec has something - to say beyond their types. `coerce`, `configuration`, `to_json` and - `canonical` are written once here against what the fields say, and - the class variables below -- `member_types`, `nested_members`, - `value_checks`, `member_rules` -- are that reading, compiled at class - creation: nothing declares them. + A subclass writes its fields and, where the spec has something to + say beyond their types, a `__post_init__` that collects every problem + and raises `MetadataValidationError` once -- so `BloscCodec(clevel=99)` + raises, and `coerce` reports the same problems instead. `coerce`, + `configuration`, `to_json` and `canonical` are written once here + against what the fields say, and the class variables below -- + `member_types`, `nested_members` -- are that reading, compiled at + class creation: nothing declares them. """ extension_point: ClassVar[ExtensionPointField | None] = None @@ -790,7 +728,7 @@ class MetadataEntity(Generic[JSONT_co]): each one's type implies. `coerce` reads a configuration against it member by member, so one member that cannot be read costs that member and not the rest. An annotation the compiler cannot read is refused - at class creation; `register_check` teaches it the shape. The public + at class creation; the field is written as a shape JSON takes. The public JSON TypedDict is held to the same keys by `tests/v3/test_entities.py`. """ @@ -803,24 +741,6 @@ class MetadataEntity(Generic[JSONT_co]): contains entities writes nothing for any of that. """ - value_checks: ClassVar[Mapping[str, TypeCheck]] = MappingProxyType({}) - """The value rules the field annotations state, member by member. - - A bound in an `Annotated` -- `level: Annotated[int, Interval(ge=0, - le=9)]`, `chunk_shape: tuple[Annotated[int, Ge(1)], ...]` -- becomes - a check here at class creation, located at the member and, for an - element, at its position. Runs with `value_problems`, after the type - checks, on both the reading path and the constructor. - """ - - member_rules: ClassVar[Mapping[str, tuple[MemberRule, ...]]] = MappingProxyType({}) - """The `@validates` rules, by the member each is about. - - Collected at class creation from the class and its ancestors, the - nearest definition of a name winning. Run after the annotation bounds - and before `value_problems`, on both paths. - """ - configuration_required: ClassVar[bool] = False """Whether the bare-name spelling says too little for this entity. @@ -894,11 +814,15 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # `typesize` requirement reads `shuffle`. Judging around the # hole would be guessing, so the type problems stand alone. return None, found - found = (*found, *within((), cls._judge_values(members))) + try: + entity = cls(**members) + except MetadataValidationError as refused: + # `__post_init__` found values the spec disallows: reported + # rather than raised, located under the configuration. + return None, (*found, *within((), refused.problems)) if any(entry.kind != "unknown_key" for entry in found): return None, found - # Already asked, so do not ask again on the way in. - return cls.unchecked(**members), found + return entity, found @final def canonical(self) -> Self: @@ -969,83 +893,11 @@ def configuration(self) -> dict[str, object]: members[name] = render_nested(annotation, members[name]) return deepcopy(members) - value_problems: ClassVar[ValueRoutine] = staticmethod(_no_value_problems) - """What the spec disallows among the members taken together. - - The third of three places a value rule lives, for the rule that - reads two members at once -- blosc's `typesize` against its - `shuffle`. A bound on one member is on the field; a rule about one - member is a `@validates` staticmethod. A routine rather than a - method, because judging values does not need an entity -- and - needing one would mean an invalid one had been built. Takes - `Unpack[Configuration]`: the same spelling the constructor - takes, receiving only the members that are present. - - Typed loosely here because the base does not know any entity's - configuration, and saying so is the truth. Call a specific routine by - its own name to have the arguments checked. - - Locations are relative to the entity's `configuration`. - """ - - @final - def __post_init__(self) -> None: - """Refuse to exist with values the spec disallows. - - So an instance is the value guarantee, not just the type one: - `BloscCodec(clevel=99)` raises rather than serializing a document - no reader will accept. `coerce` asks `value_problems` first and - reports, so reading a bad document still returns problems rather - than raising, and `unchecked` is the door for a caller that has - already asked. - """ - found = type(self)._judge_values(self._members()) - if len(found) != 0: - raise MetadataValidationError(found) - - @classmethod - def _judge_values(cls, members: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - """Every value problem among the members. - - The bounds the annotations state first, then the `@validates` - rules, member by member, then whatever `value_problems` has to - say about the members together -- one routine for the reading - path and the constructor, so the two cannot disagree. - """ - from_annotations = [ - found - for name, check in cls.value_checks.items() - if name in members - for found in check(members[name], (name,)) - ] - from_rules = [ - ValidationProblem((member, *found.loc), found.message, found.kind) - for member, member_rules in cls.member_rules.items() - if member in members - for rule in member_rules - for found in rule(members[member]) - ] - return (*from_annotations, *from_rules, *cls.value_problems(**members)) - - def _members(self) -> dict[str, object]: - """Every member this entity holds, unrendered. - - The dataclass's own fields, which is what `value_problems` - judges: a member is a member whether or not the JSON spells it - as a configuration key. The raw-bytes family is the case that - separates the two -- its width lives in its name, so it has a - field and no configuration at all. - """ - return { - field_.name: value - for field_ in fields(self) - if (value := getattr(self, field_.name)) is not UNSET - } - def _configuration_members(self) -> dict[str, object]: """The members a configuration object would spell out. - `_members` minus anything the envelope carries some other way. + Every field but one the envelope carries some other way -- the + `r` width, which lives in the name. """ return { key: value @@ -1053,39 +905,6 @@ def _configuration_members(self) -> dict[str, object]: if (value := getattr(self, key)) is not UNSET } - @classmethod - def unchecked(cls, **members: object) -> Self: - """This entity, without asking whether its values are allowed. - - For a caller that has already asked -- `coerce` does, so that it - can report the answer instead of raising it. Named so that - choosing it is deliberate. - - Unchecked means *value*-unchecked. A member this entity does not - declare, or one with neither a value nor a default, is still a - `TypeError`: those produce an entity that cannot be repred, - compared or hashed, which no caller is asking for. - """ - declared = {field_.name: field_ for field_ in fields(cls)} - unknown = sorted(members.keys() - declared.keys()) - if len(unknown) != 0: - msg = f"{cls.__name__} has no member(s) {', '.join(unknown)}" - raise TypeError(msg) - entity = object.__new__(cls) - for name, field_ in declared.items(): - if name in members: - object.__setattr__(entity, name, members[name]) - elif field_.default is not MISSING: - object.__setattr__(entity, name, field_.default) - elif field_.default_factory is not MISSING: # pragma: no cover - none today - object.__setattr__(entity, name, field_.default_factory()) - else: - # Leaving it unset would give an entity whose `repr`, - # `==` and `hash` raise `AttributeError` on access. - msg = f"{cls.__name__} is missing a value for {name!r}" - raise TypeError(msg) - return entity - def to_json(self) -> JSONT_co: """This entity as a document would write it. @@ -1233,27 +1052,19 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "DATA_TYPE", "FROM_NAME", "STORAGE_TRANSFORMERS", - "CheckCompiler", "ChunkGridEntity", "CodecEntity", "CodecKind", "Coerced", "DataTypeEntity", "ExtensionPointField", - "Ge", - "Gt", - "Interval", "JSONT_co", - "Le", "Loc", - "Lt", - "MemberRule", "MemberTypes", "MetadataEntity", "Opaque", "StorageClass", "TypeCheck", - "ValueRoutine", "coerce_members", "is_bool", "is_entity", @@ -1266,8 +1077,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "named_configuration", "one_of", "problem", - "register_check", "sequence_of", - "validates", "within", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index ad59d3da2b..952199a541 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -5,14 +5,14 @@ """ from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Annotated, ClassVar, Final, Literal, NotRequired, Self, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import TypedDict -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, - Ge, + Loc, is_integer, problem, ) @@ -43,15 +43,6 @@ pairs. """ -_PositiveExtent = Annotated[int, Ge(1)] -_PositiveDimSpec = ( - _PositiveExtent | tuple[_PositiveExtent | tuple[_PositiveExtent, _PositiveExtent], ...] -) -"""`RectilinearDimSpec` as the entity holds it: every extent, and every -run-length count, at least one. The same type to a type checker; the -bounds are what the reading path judges. -""" - class RectilinearChunkGridConfiguration(TypedDict, closed=True): """Configuration for the rectilinear chunk grid.""" @@ -78,6 +69,10 @@ class RectilinearChunkGridObject(TypedDict, closed=True): """ +def _not_positive(loc: Loc, value: int) -> tuple[ValidationProblem, ...]: + return problem(loc, f"expected an integer >= 1, got {value}", "invalid_value") + + def canonical_dim_spec(spec: RectilinearDimSpec) -> RectilinearDimSpec: """One dimension's chunk sizes in their simplest equivalent form. @@ -176,10 +171,29 @@ class RectilinearChunkGrid(ChunkGridEntity[RectilinearChunkGridMetadata]): """The `rectilinear` chunk grid, coerced from its metadata.""" kind: Literal["inline"] - chunk_shapes: tuple[_PositiveDimSpec, ...] + chunk_shapes: tuple[RectilinearDimSpec, ...] identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME + def __post_init__(self) -> None: + """Every extent, and every run's length and count, is at least 1.""" + found: list[ValidationProblem] = [] + for axis, spec in enumerate(self.chunk_shapes): + if isinstance(spec, int): + if spec < 1: + found.extend(_not_positive(("chunk_shapes", axis), spec)) + continue + for index, entry in enumerate(spec): + if isinstance(entry, int): + if entry < 1: + found.extend(_not_positive(("chunk_shapes", axis, index), entry)) + continue + for position, value in enumerate(entry): + if value < 1: + found.extend(_not_positive(("chunk_shapes", axis, index, position), value)) + if len(found) != 0: + raise MetadataValidationError(found) + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index e10a8f3cb1..dc791ef697 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -5,14 +5,13 @@ """ from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, ClassVar, Final, Literal, NotRequired, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast from typing_extensions import TypedDict -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, - Ge, problem, ) from zarr_metadata.v3._parts import ChunkGrid @@ -65,10 +64,24 @@ class RegularChunkGridObject(TypedDict, closed=True): class RegularChunkGrid(ChunkGridEntity[RegularChunkGridMetadata]): """The `regular` chunk grid, coerced from its metadata.""" - chunk_shape: tuple[Annotated[int, Ge(1)], ...] + chunk_shape: tuple[int, ...] identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME + def __post_init__(self) -> None: + found: list[ValidationProblem] = [] + for index, extent in enumerate(self.chunk_shape): + if extent < 1: + found.extend( + problem( + ("chunk_shape", index), + f"expected an integer >= 1, got {extent}", + "invalid_value", + ) + ) + if len(found) != 0: + raise MetadataValidationError(found) + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" if not isinstance(array_shape, (list, tuple)): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index f2480f8f7c..ccbbd176ad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,17 +5,15 @@ """ from dataclasses import dataclass, replace -from typing import Annotated, ClassVar, Final, Literal, NotRequired, Self +from typing import ClassVar, Final, Literal, NotRequired, Self -from typing_extensions import TypedDict, Unpack +from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - Ge, - Interval, problem, ) @@ -100,9 +98,9 @@ class BloscCodec(CodecEntity[BloscCodecMetadata]): """ cname: BloscCName - clevel: Annotated[int, Interval(ge=0, le=9)] + clevel: int shuffle: BloscShuffle - blocksize: Annotated[int, Ge(0)] + blocksize: int typesize: int | UNSET = UNSET identifier: ClassVar[str] = BLOSC_CODEC_NAME @@ -112,35 +110,49 @@ class BloscCodec(CodecEntity[BloscCodecMetadata]): # Every member is required but `typesize`, which only means something # when shuffling; `problems` is where that conditional lives. - @staticmethod - def value_problems( - **members: Unpack[BloscCodecConfiguration], - ) -> tuple[ValidationProblem, ...]: - """`typesize` against `shuffle`: required, and positive, only where it counts. + def __post_init__(self) -> None: + """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. Under `noshuffle` the spec says of `typesize` that "the value is - ignored", and `canonical` drops it. A rule over two members, which - is what this routine is for; the bounds on `clevel` and - `blocksize` are on the fields. + ignored", and `simplified` drops it; under either shuffle it is + required, and positive. """ - shuffle = members["shuffle"] - typesize = members.get("typesize") found: list[ValidationProblem] = [] - if typesize is not None and shuffle != BLOSC_NO_SHUFFLE and typesize < 1: + if not 0 <= self.clevel <= 9: found.extend( problem( - ("typesize",), f"expected a positive integer, got {typesize}", "invalid_value" + ("clevel",), + f"expected an integer in [0, 9], got {self.clevel}", + "invalid_value", ) ) - if shuffle != BLOSC_NO_SHUFFLE and typesize is None: + if self.blocksize < 0: found.extend( problem( - ("typesize",), - f"typesize is required when shuffle is {shuffle!r}", - "missing_key", + ("blocksize",), + f"expected an integer >= 0, got {self.blocksize}", + "invalid_value", ) ) - return tuple(found) + if self.shuffle != BLOSC_NO_SHUFFLE: + if self.typesize is UNSET: + found.extend( + problem( + ("typesize",), + f"typesize is required when shuffle is {self.shuffle!r}", + "missing_key", + ) + ) + elif self.typesize < 1: + found.extend( + problem( + ("typesize",), + f"expected a positive integer, got {self.typesize}", + "invalid_value", + ) + ) + if len(found) != 0: + raise MetadataValidationError(found) def simplified(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 324fdcd5e7..9774f298b7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -5,14 +5,15 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict +from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - Interval, + problem, ) GZIP_CODEC_NAME: Final = "gzip" @@ -69,8 +70,16 @@ class GzipCodecObject(TypedDict, closed=True): class GzipCodec(CodecEntity[GzipCodecMetadata]): """The `gzip` codec, coerced from its metadata.""" - level: Annotated[int, Interval(ge=0, le=9)] + level: int identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" + + def __post_init__(self) -> None: + if not 0 <= self.level <= 9: + raise MetadataValidationError( + problem( + ("level",), f"expected an integer in [0, 9], got {self.level}", "invalid_value" + ) + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index d9d2ab9ab7..f3d9f4d359 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -11,12 +11,11 @@ from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, problem, - validates, ) from zarr_metadata.v3._parts import ArrayParts @@ -89,17 +88,7 @@ class ScaleOffsetCodec(CodecEntity[ScaleOffsetCodecMetadata]): identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME kind: ClassVar[CodecKind] = "array_array" - def transition(self, incoming: ArrayParts) -> ArrayParts | None: - """The same array, element for element. - - The registry entry removed the `astype` field, so this codec no - longer changes the element type -- only the values. - """ - return incoming - - @staticmethod - @validates("offset", "scale") - def _is_a_scalar(value: JSONValue) -> tuple[ValidationProblem, ...]: + def __post_init__(self) -> None: """Each value is a scalar of the array's type, so neither is null. The registry says each is "JSON-encoded per the input array's @@ -107,6 +96,18 @@ def _is_a_scalar(value: JSONValue) -> tuple[ValidationProblem, ...]: Which scalar it should be needs the data type, so that part is the document's question, not this codec's. """ - if value is None: - return problem((), "expected a scalar, got null", "invalid_value") - return () + found: list[ValidationProblem] = [] + if self.offset is None: + found.extend(problem(("offset",), "expected a scalar, got null", "invalid_value")) + if self.scale is None: + found.extend(problem(("scale",), "expected a scalar, got null", "invalid_value")) + if len(found) != 0: + raise MetadataValidationError(found) + + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + """The same array, element for element. + + The registry entry removed the `astype` field, so this codec no + longer changes the element type -- only the values. + """ + return incoming diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 6a4ceb7465..4667a4c691 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -5,18 +5,17 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - Ge, Opaque, problem, ) @@ -104,7 +103,7 @@ class ShardingIndexedCodec(CodecEntity[ShardingIndexedCodecMetadata]): itself an entity, read the same way this one was. """ - chunk_shape: tuple[Annotated[int, Ge(1)], ...] + chunk_shape: tuple[int, ...] codecs: tuple[CodecEntity | Opaque, ...] index_codecs: tuple[CodecEntity | Opaque, ...] index_location: ShardingIndexLocation | UNSET = UNSET @@ -113,6 +112,20 @@ class ShardingIndexedCodec(CodecEntity[ShardingIndexedCodecMetadata]): variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "array_bytes" + def __post_init__(self) -> None: + found: list[ValidationProblem] = [] + for index, extent in enumerate(self.chunk_shape): + if extent < 1: + found.extend( + problem( + ("chunk_shape", index), + f"expected an integer >= 1, got {extent}", + "invalid_value", + ) + ) + if len(found) != 0: + raise MetadataValidationError(found) + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """This shard against the array reaching it, and its two pipelines. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 2ef20918c6..3d8edb3a4e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -9,12 +9,11 @@ from typing_extensions import TypedDict -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, problem, - validates, ) from zarr_metadata.v3._parts import ArrayParts @@ -72,19 +71,20 @@ class TransposeCodec(CodecEntity[TransposeCodecMetadata]): identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME kind: ClassVar[CodecKind] = "array_array" - @staticmethod - @validates("order") - def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: + def __post_init__(self) -> None: """`order` must permute its own axes. Whether it permutes the *array's* axes is a different question -- it needs the array's rank -- and the rules layer asks that one. """ - if sorted(order) != list(range(len(order))): - return problem( - (), f"expected a permutation of 0..{len(order) - 1}, got {order!r}", "invalid_value" + if sorted(self.order) != list(range(len(self.order))): + raise MetadataValidationError( + problem( + ("order",), + f"expected a permutation of 0..{len(self.order) - 1}, got {self.order!r}", + "invalid_value", + ) ) - return () def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index af652a773d..b853475973 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -7,15 +7,16 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( CodecEntity, CodecKind, - Interval, + problem, ) ZSTD_CODEC_NAME: Final = "zstd" @@ -77,9 +78,19 @@ class ZstdCodecObject(TypedDict, closed=True): class ZstdCodec(CodecEntity[ZstdCodecMetadata]): """The `zstd` codec, coerced from its metadata.""" - level: Annotated[int, Interval(ge=ZSTD_MIN_LEVEL, le=ZSTD_MAX_LEVEL)] + level: int checksum: bool | UNSET = UNSET identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True kind: ClassVar[CodecKind] = "bytes_bytes" + + def __post_init__(self) -> None: + if not ZSTD_MIN_LEVEL <= self.level <= ZSTD_MAX_LEVEL: + raise MetadataValidationError( + problem( + ("level",), + f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {self.level}", + "invalid_value", + ) + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index b35ec45ce0..e2d4003bd7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -5,13 +5,14 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict +from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( - Interval, StorageClass, + problem, ) from zarr_metadata.v3.data_type._families import ( NUMPY_TIME_MAX_SCALE_FACTOR, @@ -73,7 +74,18 @@ class NumpyDatetime64DataType(NumpyTimeDataType[NumpyDatetime64]): """The `numpy.datetime64` data type, coerced from its metadata.""" unit: NumpyTimeUnit - scale_factor: Annotated[int, Interval(ge=1, le=NUMPY_TIME_MAX_SCALE_FACTOR)] + scale_factor: int scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME + + def __post_init__(self) -> None: + if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + raise MetadataValidationError( + problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " + f"got {self.scale_factor}", + "invalid_value", + ) + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 4e596d13ef..03fc1d72e3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -5,13 +5,14 @@ """ from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, Literal, NotRequired +from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import ReadOnly, TypedDict +from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( - Interval, StorageClass, + problem, ) from zarr_metadata.v3.data_type._families import ( NUMPY_TIME_MAX_SCALE_FACTOR, @@ -76,7 +77,18 @@ class NumpyTimedelta64DataType(NumpyTimeDataType[NumpyTimedelta64]): """The `numpy.timedelta64` data type, coerced from its metadata.""" unit: NumpyTimeUnit - scale_factor: Annotated[int, Interval(ge=1, le=NUMPY_TIME_MAX_SCALE_FACTOR)] + scale_factor: int scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME + + def __post_init__(self) -> None: + if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + raise MetadataValidationError( + problem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " + f"got {self.scale_factor}", + "invalid_value", + ) + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index f482ba740e..628d833a15 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -12,9 +12,7 @@ from dataclasses import dataclass from typing import Annotated, ClassVar, Final, NewType, Self -from typing_extensions import TypedDict, Unpack - -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( FROM_NAME, Coerced, @@ -99,12 +97,6 @@ def _name_problems(name: str) -> tuple[ValidationProblem, ...]: return () -class RawBytesMembers(TypedDict): - """A raw-bytes type's members: the spelling, which carries the width.""" - - data_type_name: str - - @dataclass(frozen=True) class RawBytesDataType(DataTypeEntity[RawBytesDataTypeName]): """An `r` raw-bytes data type, coerced from its metadata. @@ -147,20 +139,19 @@ def coerce(cls, value: object, context: object) -> Coerced[Self]: # its fill values are still judged. Returning nothing here let # a stray key hide every other problem in the document. found = problem(("configuration",), "'r' takes no configuration", "unknown_key") - found = (*found, *cls.value_problems(data_type_name=name)) + try: + entity = cls(data_type_name=name) + except MetadataValidationError as refused: + return None, (*found, *refused.problems) if any(entry.kind != "unknown_key" for entry in found): return None, found - return cls.unchecked(data_type_name=name), found - - @staticmethod - def value_problems(**members: Unpack[RawBytesMembers]) -> tuple[ValidationProblem, ...]: - """This family's validity is in its name, not in a configuration. + return entity, found - Which is the one place a member is not a configuration key, and - why `value_problems` judges the entity's fields rather than its - configuration: there is no configuration here to judge. - """ - return _name_problems(members["data_type_name"]) + def __post_init__(self) -> None: + """This family's validity is in its name, not in a configuration.""" + found = _name_problems(self.data_type_name) + if len(found) != 0: + raise MetadataValidationError(found) def to_json(self) -> RawBytesDataTypeName: return RawBytesDataTypeName(self.data_type_name) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index bcc4b8191b..2b7b5659dc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -11,7 +11,7 @@ from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.model._validation import ValidationProblem +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( DataTypeEntity, @@ -19,7 +19,6 @@ Opaque, StorageClass, problem, - validates, ) STRUCT_DATA_TYPE_NAME: Final = "struct" @@ -110,11 +109,7 @@ class StructDataType(DataTypeEntity[Struct]): identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" - @staticmethod - @validates("fields") - def _fields_form_a_record( - fields: tuple[StructFieldComponent, ...], - ) -> tuple[ValidationProblem, ...]: + def __post_init__(self) -> None: """Names exist, are non-empty and distinct; types are fixed-size. A fill value addresses fields by name, and a record's layout is @@ -123,19 +118,25 @@ def _fields_form_a_record( are allowed. """ found: list[ValidationProblem] = [] - if len(fields) == 0: - found.extend(problem((), "expected at least one struct field", "invalid_value")) + if len(self.fields) == 0: + found.extend( + problem(("fields",), "expected at least one struct field", "invalid_value") + ) seen: dict[str, int] = {} - for index, field in enumerate(fields): + for index, field in enumerate(self.fields): if field.name == "": found.extend( - problem((index, "name"), "expected a non-empty field name", "invalid_value") + problem( + ("fields", index, "name"), + "expected a non-empty field name", + "invalid_value", + ) ) first = seen.setdefault(field.name, index) if first != index: found.extend( problem( - (index, "name"), + ("fields", index, "name"), f"duplicate field name {field.name!r}, already used by field {first}", "invalid_value", ) @@ -146,12 +147,13 @@ def _fields_form_a_record( ): found.extend( problem( - (index, "data_type"), + ("fields", index, "data_type"), "struct fields must use fixed-size data types", "invalid_value", ) ) - return tuple(found) + if len(found) != 0: + raise MetadataValidationError(found) def storage_class(self) -> StorageClass | None: """The widest class among the fields. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index e49198ad14..9d47aec43c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -48,38 +48,19 @@ class AcmeLz4Codec(CodecEntity): The fields are the only place the shape is written. Which members exist, which may be left out (the type admits `UNSET`), and how each one is type-checked are all read off the annotations -- an `int`, a `Literal` -of names, an array, a nested entity type -- and an annotation the -compiler does not read is taught to it once, with `register_check`. A -bound on a value is written on the field too, in the `annotated_types` -vocabulary: - - acceleration: Annotated[int, Interval(ge=1, le=65537)] | UNSET = UNSET - -A rule about one member that is not a bound is a `@validates` rule: a -staticmethod taking the member's value, run only when the member is -present and has the type it declared, reporting relative to the member: - - @staticmethod - @validates("order") - def _order_permutes_itself(order: tuple[int, ...]) -> tuple[ValidationProblem, ...]: - if sorted(order) != list(range(len(order))): - return problem((), f"expected a permutation, got {order!r}", "invalid_value") - return () - -A rule that reads two members together goes in a `value_problems` -staticmethod; annotate it with a TypedDict of the members so its body is -checked: - - class AcmeLz4Configuration(TypedDict, closed=True): - acceleration: NotRequired[int] - - @staticmethod - def value_problems( - **members: Unpack[AcmeLz4Configuration], - ) -> tuple[ValidationProblem, ...]: - if "acceleration" not in members: - return () - ... +of names, an array, a nested object, a nested entity type: the shapes +JSON takes, and no others. Everything finer -- a bound, a rule about a +member, members read together -- is `__post_init__`, in plain code, +collecting every problem and raising once; `coerce` reports those +instead of raising, located under the configuration: + + acceleration: int | UNSET = UNSET + + def __post_init__(self) -> None: + if self.acceleration is not UNSET and not 1 <= self.acceleration <= 65537: + raise MetadataValidationError( + problem(("acceleration",), f"expected an integer in [1, 65537], got {self.acceleration}", "invalid_value") + ) A name in no scope is not rejected -- that is what extension openness means -- so registering yours is how you get it judged rather than waved @@ -117,25 +98,18 @@ def value_problems( DATA_TYPE, FROM_NAME, STORAGE_TRANSFORMERS, - CheckCompiler, ChunkGridEntity, CodecEntity, CodecKind, Coerced, DataTypeEntity, ExtensionPointField, - Ge, - Gt, - Interval, - Le, Loc, - Lt, MemberTypes, MetadataEntity, Opaque, StorageClass, TypeCheck, - ValueRoutine, coerce_members, is_bool, is_int, @@ -146,9 +120,7 @@ def value_problems( named_configuration, one_of, problem, - register_check, sequence_of, - validates, within, ) from zarr_metadata.v3._parts import UNKNOWN_GRID, ArrayParts, ChunkGrid, Extents, shard_index_grid @@ -182,7 +154,6 @@ def value_problems( "UNKNOWN_GRID", "ArrayDocumentV3", "ArrayParts", - "CheckCompiler", "ChunkGrid", "ChunkGridEntity", "CodecEntity", @@ -195,13 +166,8 @@ def value_problems( "ExtensionPointField", "Extents", "FloatDataType", - "Ge", - "Gt", "IntegerDataType", - "Interval", - "Le", "Loc", - "Lt", "MemberTypes", "MetadataEntity", "NumpyTimeDataType", @@ -209,7 +175,6 @@ def value_problems( "PartialEntityTables", "StorageClass", "TypeCheck", - "ValueRoutine", "array_problems_v3", "as_sequence", "byte_values", @@ -226,9 +191,7 @@ def value_problems( "order_problems", "problem", "read_array_v3", - "register_check", "sequence_of", "shard_index_grid", - "validates", "within", ] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 9967c72590..b54713ba61 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -288,7 +288,6 @@ def test_all_is_grouped_and_unique() -> None: "BloscCName", "BloscShuffle", "Canonical", - "CheckCompiler", "CastOutOfRangeMode", "CastRoundingMode", "CodecKind", @@ -297,11 +296,6 @@ def test_all_is_grouped_and_unique() -> None: "MemberTypes", "Loc", "Extents", - "Ge", - "Gt", - "Interval", - "Le", - "Lt", "ExtensionPointField", "Context", "Coerced", @@ -329,7 +323,6 @@ def test_all_is_grouped_and_unique() -> None: "Struct", "StructField", "ValidationProblem", - "ValueRoutine", } ) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 5833a627ea..9ce79dfd26 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -147,25 +147,6 @@ def test_the_constructor_mirrors_the_configuration(entity: type[MetadataEntity]) assert fields == set(get_type_hints(configuration)) -@pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) -def test_the_value_routine_takes_the_members_it_will_be_given( - entity: type[MetadataEntity], -) -> None: - # `coerce` calls it as `value_problems(**members)`, which no type can - # check: members is a dict built at run time. So the correspondence - # is checked here, against the fields rather than the configuration - # -- `r` holds a member that is not a configuration key, and a - # `struct` and a `sharding_indexed` annotate a TypedDict of their own - # because `prepare` has replaced field objects with entities by then. - # Neither changes which members there are. - if entity.value_problems is MetadataEntity.value_problems: - return - # The annotation is `Unpack[X]`; X is what says which members. - (members,) = get_args(get_type_hints(entity.value_problems)["members"]) - fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} - assert set(get_type_hints(members)) == fields - - def _parts(json_type: object) -> tuple[object, ...]: return ( get_args(json_type) if get_origin(json_type) in (Union, types.UnionType) else (json_type,) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 7cc20dcedc..5877610701 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,18 +8,17 @@ import re from dataclasses import dataclass, replace -from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast +from typing import ClassVar, Literal, NotRequired, Self, cast import pytest from typing_extensions import TypedDict -from zarr_metadata.model import UNSET, MetadataValidationError, ValidationProblem +from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import ( canonicalize_array_metadata_v3, validate_array_metadata_v3, ) from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -from zarr_metadata.v3._compile import _CHECK_COMPILERS from zarr_metadata.v3._entity import json_type_of from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.gzip import GzipCodec @@ -35,15 +34,11 @@ Context, DataTypeEntity, IntegerDataType, - Interval, - Loc, MetadataEntity, Opaque, StorageClass, named_configuration, problem, - register_check, - validates, ) ACME_MAX_ACCELERATION = 65537 @@ -53,12 +48,22 @@ class AcmeLz4Codec(CodecEntity): """A third-party compressor.""" - acceleration: Annotated[int, Interval(ge=1, le=ACME_MAX_ACCELERATION)] | UNSET = UNSET + acceleration: int | UNSET = UNSET identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" variable_size: ClassVar[bool] = True + def __post_init__(self) -> None: + if self.acceleration is not UNSET and not 1 <= self.acceleration <= ACME_MAX_ACCELERATION: + raise MetadataValidationError( + problem( + ("acceleration",), + f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {self.acceleration}", + "invalid_value", + ) + ) + @dataclass(frozen=True) class AcmeFloat8DataType(DataTypeEntity): @@ -262,21 +267,6 @@ def test_a_reader_can_choose_its_own_scope() -> None: assert in_scope.acceleration == 4 -def test_error_an_entity_may_not_validate_in_post_init() -> None: - # `coerce` builds through `unchecked`, which never reaches - # `__post_init__`, so a rule there holds for a hand-built entity and - # is silently absent for every entity read from a document. - with pytest.raises(TypeError, match="`unchecked` does not reach"): - - @dataclass(frozen=True) - class Eager(CodecEntity): # pyright: ignore[reportUnusedClass] - identifier: ClassVar[str] = "acme.eager" - kind: ClassVar[CodecKind] = "bytes_bytes" - - def __post_init__(self) -> None: # pyright: ignore[reportIncompatibleMethodOverride] - raise AssertionError - - def test_error_a_field_may_not_shadow_a_class_variable() -> None: # A field of that name goes into the configuration and into the JSON, # while the class variable it shadows is what the rest of the layer @@ -325,7 +315,7 @@ def coerce(cls, value: object, context: object) -> Coerced[Self]: name, _, _ = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), "expected an 'acme.fixedN' data type") - return cls.unchecked(data_type_name=name), () + return cls(data_type_name=name), () def to_json(self) -> ZarrV3MetadataFieldJSON: return cast("ZarrV3MetadataFieldJSON", self.data_type_name) @@ -355,7 +345,7 @@ def test_error_a_member_needs_a_check_from_somewhere() -> None: # An annotation outside the shapes `check_for` compiles implies no # check, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. - with pytest.raises(TypeError, match="annotation of inner; teach the compiler that shape"): + with pytest.raises(TypeError, match="annotation of inner; a field is one of the shapes JSON"): @dataclass(frozen=True) class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] @@ -367,7 +357,7 @@ class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] @pytest.mark.parametrize( "name", - ["member_types", "configuration_required", "nested_members", "value_checks", "member_rules"], + ["member_types", "configuration_required", "nested_members"], ) def test_error_a_derived_class_variable_may_not_be_declared(name: str) -> None: # Each is read off the fields at class creation, and a declaration @@ -502,7 +492,7 @@ class Vague(CodecEntity): # pyright: ignore[reportUnusedClass] kind: ClassVar[CodecKind] = "bytes_bytes" -# A third-party rule about one member, written as a `@validates` rule. +# A third-party rule about a member, written in `__post_init__`. @dataclass(frozen=True) class AcmeBlockCodec(CodecEntity): """A codec whose block size must be a power of two.""" @@ -512,19 +502,17 @@ class AcmeBlockCodec(CodecEntity): identifier: ClassVar[str] = "acme.block" kind: ClassVar[CodecKind] = "bytes_bytes" - @staticmethod - @validates("block") - def _block_is_a_power_of_two(block: int) -> tuple[ValidationProblem, ...]: - if block < 1 or block & (block - 1) != 0: - return problem((), f"expected a power of two, got {block}", "invalid_value") - return () + def __post_init__(self) -> None: + if self.block < 1 or self.block & (self.block - 1) != 0: + raise MetadataValidationError( + problem(("block",), f"expected a power of two, got {self.block}", "invalid_value") + ) -def test_a_rule_about_one_member_is_a_validates_rule() -> None: - # The rule receives the typed member, only when present, and reports - # relative to it: the location is supplied, and no `**members` is - # unpacked by hand. The declared signature survives, so a call by - # name is checked. +def test_a_rule_about_a_member_is_post_init() -> None: + # The rule runs on the typed members and reports relative to the + # configuration; `coerce` catches what it raises and locates it in + # the document, and the constructor raises it as it is. scope = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeBlockCodec.identifier: AcmeBlockCodec}) codec, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": 64}}) assert problems == () @@ -542,83 +530,6 @@ def test_a_rule_about_one_member_is_a_validates_rule() -> None: assert [p.kind for p in problems] == ["invalid_type"] -def test_error_a_validates_rule_must_name_a_field() -> None: - with pytest.raises(TypeError, match="`@validates\\('blocc'\\)` names no field"): - - @dataclass(frozen=True) - class Misspelt(CodecEntity): # pyright: ignore[reportUnusedClass] - block: int - - identifier: ClassVar[str] = "acme.misspelt" - kind: ClassVar[CodecKind] = "bytes_bytes" - - @staticmethod - @validates("blocc") - def _rule(block: int) -> tuple[ValidationProblem, ...]: - return () - - -# An annotation shape the compiler does not read, taught to it from outside. -class Hex(str): - """A hex digest: a `str` to the type checker, its own class at run time.""" - - __slots__ = () - - -def _is_hex(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, str) or any(c not in "0123456789abcdef" for c in value): - return problem(loc, f"expected lowercase hex digits, got {value!r}") - return () - - -def test_a_third_party_can_teach_the_compiler_a_shape() -> None: - # A `str` subclass is a real case: to the type checker `Hex` is a - # `str`, but at run time it is a class `check_for` has no registration - # for, so an entity using it is refused -- until one is registered, - # through the same door the built-in shapes came through. - with pytest.raises(TypeError, match="no check can be read off the annotation of digest"): - - @dataclass(frozen=True) - class Unregistered(CodecEntity): # pyright: ignore[reportUnusedClass] - digest: Hex - - identifier: ClassVar[str] = "acme.unregistered" - kind: ClassVar[CodecKind] = "bytes_bytes" - - def is_hex_annotation(annotation: object) -> bool: - return annotation is Hex - - register_check(is_hex_annotation, lambda annotation: _is_hex) - try: - - @dataclass(frozen=True) - class AcmeDigestCodec(CodecEntity): - digest: Hex - - identifier: ClassVar[str] = "acme.digest" - kind: ClassVar[CodecKind] = "bytes_bytes" - - scope = CORE_AND_EXTENSIONS.extended_with( - codecs={AcmeDigestCodec.identifier: AcmeDigestCodec} - ) - codec, problems = scope.coerce( - "codecs", {"name": "acme.digest", "configuration": {"digest": "c0ffee"}} - ) - assert problems == () - assert isinstance(codec, AcmeDigestCodec) - _, problems = scope.coerce( - "codecs", {"name": "acme.digest", "configuration": {"digest": "C0FFEE"}} - ) - assert [(p.loc, p.kind) for p in problems] == [ - (("configuration", "digest"), "invalid_type") - ] - finally: - # A registration is process-wide; leave the compiler as it was found. - _CHECK_COMPILERS[:] = [ - entry for entry in _CHECK_COMPILERS if entry.predicate is not is_hex_annotation - ] - - def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: # A required member means the entity is always written as an object, # so naming a bare-name type for it is a promise `to_json` would break. From 3e310ae6e9722739aa1c3b9d95ec8494e6b6fd87 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 14:36:29 +0200 Subject: [PATCH 074/107] fix(zarr-metadata): the first block from the adversarial review Behaviour: `coerce` no longer builds an entity around a member that could not be read -- `__post_init__` was judging the hole and reporting a second, contradictory problem at the same location -- and resolves nested entities whatever else it found, so their problems are reported in the same pass (40k-document differential: 0 lost, 1,191 gained, 0 verdict flips). The named JSON type is held to key by key: every name it lists is one the entity accepts, its configuration keys are the members with the members' requiredness, and the spellings it admits are the ones the members make the entity write. `FROM_NAME` fields are filled from the envelope's name by the base, so the raw-bytes family needs no reader of its own. `ArrayDocumentV3.to_json` writes back only the fields the document had, instead of inventing nulls. A slotted dataclass is compiled once. A bare `ClassVar` is a class variable. `float` is a shape: a JSON number. Refusals, each for a mistake that type-checked and then misbehaved somewhere that did not name the class: an entity declaring fields without `@dataclass` (at registration, the earliest place that can see it); a nested field that does not admit `Opaque`, which is what it holds when the name is out of scope; an `array_array` codec leaving `transition` at its default, which silenced every rule after it; a `Literal`-typed class variable set to a value outside the listed ones (`scalar_storage = "sixteen_bytes"` disabled the endian rule); and a list of `problem()` tuples handed to `MetadataValidationError`, which passed the constructor and failed inside `coerce`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/model/_validation.py | 12 + .../src/zarr_metadata/v3/_checks.py | 37 +-- .../src/zarr_metadata/v3/_compile.py | 23 +- .../src/zarr_metadata/v3/_document.py | 4 +- .../src/zarr_metadata/v3/_entity.py | 310 ++++++++++++++---- .../src/zarr_metadata/v3/_registry.py | 12 + .../src/zarr_metadata/v3/data_type/raw.py | 24 +- .../zarr-metadata/tests/v3/test_entities.py | 77 ++--- .../tests/v3/test_extension_api.py | 189 +++++++++-- 9 files changed, 523 insertions(+), 165 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 27becad824..5c77338672 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -84,6 +84,18 @@ class MetadataValidationError(ValueError): def __init__(self, problems: Sequence[ValidationProblem]) -> None: self.problems = tuple(problems) + for entry in self.problems: + # The type says so; the check is for the trap the type cannot + # close: `problem()` returns a one-element tuple, and a list + # of those passes here and fails far away, where a `loc` is + # read off it. + if not isinstance(entry, ValidationProblem): # pyright: ignore[reportUnnecessaryIsInstance] + msg = ( + f"MetadataValidationError takes ValidationProblem values, got " + f"{type(entry).__name__}; `problem()` returns a tuple of them, so collect " + "with `extend`, not `append`" + ) + raise TypeError(msg) super().__init__("\n".join(str(problem) for problem in self.problems)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py index 477611515b..73cc75322e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py @@ -80,6 +80,13 @@ def is_bool(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: return () +def is_number(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """A JSON number: an `int` or a `float`, and not a `bool`.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return problem(loc, f"expected a number, got {value!r}") + return () + + def is_json_value(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: """Any JSON value at all -- the widest type a member can declare.""" if not is_json(value): @@ -147,25 +154,18 @@ def _as_tuples(value: object) -> object: def coerce_members( configuration: Mapping[str, object], types: MemberTypes -) -> tuple[dict[str, object], tuple[ValidationProblem, ...], frozenset[str]]: +) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: """The members `types` declares, taken from `configuration`. - Returns what was accepted, every problem found, and the names of the - required members that could not be read. Three kinds of problem, and - they differ in that last part: - - - a key the entity does not declare says the value carries something - extra, not that it is wrong; - - an *optional* member of the wrong type leaves that member absent, - and everything else about the entity is still readable -- a bad - `index_location` says nothing about whether a shard's pipelines - are well formed, and silencing them would lose a real judgment; - - a *required* member missing or of the wrong type does stop it. - There is no honest reading of a `blosc` whose level is a string. + Returns what was read and every problem found. A key the entity does + not declare says the value carries something extra, not that it is + wrong, so the member it sits beside is still read; a member of the + wrong type, or a required one missing, is reported and left out -- + and an entity is never built around the hole, because its rules are + written over a whole configuration. """ problems: list[ValidationProblem] = [] members: dict[str, object] = {} - unreadable: set[str] = set() for key in configuration: if key not in types: problems.extend( @@ -177,21 +177,15 @@ def coerce_members( problems.extend( problem(("configuration", key), f"missing required key {key!r}", "missing_key") ) - unreadable.add(key) continue # Normalized before the check, so a check only ever sees the tuples # the TypedDicts declare -- never the lists raw JSON arrives as. value = _as_tuples(configuration[key]) found = check(value, ("configuration", key)) problems.extend(found) - # An unknown key says the value carries something extra, not that - # it is the wrong type -- so the member is still readable, and - # dropping it here would make `to_json` lose what was written. if all(entry.kind == "unknown_key" for entry in found): members[key] = value - elif required: - unreadable.add(key) - return members, tuple(problems), frozenset(unreadable) + return members, tuple(problems) def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: @@ -260,6 +254,7 @@ def named_configuration( "is_integer", "is_json_value", "is_metadata_field", + "is_number", "is_str", "named_configuration", "one_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py index 9aef99bd41..fb7160dbfd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py @@ -50,6 +50,7 @@ is_integer, is_json_value, is_metadata_field, + is_number, is_str, object_of, one_of, @@ -176,6 +177,8 @@ def describe(annotation: object) -> str: return "a metadata field" if inner is int: return "an integer" + if inner is float: + return "a number" if inner is bool: return "a boolean" if inner is str: @@ -214,6 +217,8 @@ def shape_of(annotation: object) -> str | None: return "field" if inner is int: return "int" + if inner is float: + return "number" if inner is bool: return "bool" if inner is str: @@ -238,6 +243,8 @@ def has_shape(shape: str | None, value: object) -> bool: return True if shape == "int": return is_integer(value) + if shape == "number": + return not isinstance(value, bool) and isinstance(value, (int, float)) if shape == "bool": return isinstance(value, bool) if shape == "str": @@ -391,7 +398,7 @@ def check_for(annotation: object) -> TypeCheck | None: """The type check a field annotation implies, or None if it implies none. A small compiler over the shapes JSON takes, and no others: the - scalars, a `Literal` of names, arrays homogeneous or fixed, unions of + scalars (`int`, `float` for any number, `bool`, `str`), a `Literal` of names, arrays homogeneous or fixed, unions of those, a nested object described by a TypedDict or a record dataclass, an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested metadata field -- an entity type, with or @@ -408,6 +415,8 @@ def check_for(annotation: object) -> TypeCheck | None: return is_metadata_field if inner is int: return is_int + if inner is float: + return is_number if inner is bool: return is_bool if inner is str: @@ -433,6 +442,15 @@ def check_for(annotation: object) -> TypeCheck | None: return None +def envelope_members(cls: type) -> tuple[str, ...]: + """The fields `FROM_NAME` marks: carried by the envelope's name, not by a configuration key.""" + return tuple( + name + for name, annotation in field_hints(cls).items() + if any(entry is FROM_NAME for entry in strip_annotation(annotation)[1]) + ) + + def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], list[str]]: """The member table an entity's own fields describe. @@ -475,7 +493,7 @@ def is_class_var(annotation: object) -> bool: if isinstance(annotation, str): stripped = annotation.strip() return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) - return get_origin(annotation) is ClassVar + return annotation is ClassVar or get_origin(annotation) is ClassVar def declared_class_vars(cls: type) -> dict[str, type]: @@ -501,6 +519,7 @@ def declared_class_vars(cls: type) -> dict[str, type]: "derive_member_types", "describe", "element_annotations", + "envelope_members", "field_hints", "fixed_tuple", "has_shape", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 9c966c7bc9..17e5bdfcf2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -119,10 +119,12 @@ def to_json(self) -> dict[str, object]: are not extension points come back exactly as the document had them. Ask `canonical` first for the simplest equivalent spelling. """ + # Only the fields the document wrote: an absent one was read as an + # `Opaque` standing in, and writing it back would invent a null. rendered = { name: render_nested(annotation, getattr(self, name)) for name, annotation in field_hints(type(self)).items() - if contains_entity(annotation) + if contains_entity(annotation) and name in self.document } return {**self.document, **rendered} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index a1cb2bf956..cb4922152f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -88,6 +88,7 @@ declared_class_vars, derive_member_types, element_annotations, + envelope_members, field_hints, has_shape, is_class_var, @@ -122,9 +123,11 @@ class GzipCodec(CodecEntity[GzipCodecMetadata]): ... """The entity if it could be built, and every problem found. One direction holds: no entity means at least one problem. The converse -does not -- a survivable problem (an unknown key, an optional member of -the wrong type) comes back *with* the entity, because the entity is -still readable and saying so is more useful than refusing. +does not -- a survivable problem, an unknown key, comes back *with* the +entity, because the entity is still readable and saying so is more +useful than refusing. A member of the wrong type is not survivable: +the entity's rules are written over a whole configuration, and an +entity is never built around a hole. So test `entity is None` to decide whether to go on reading, and test the problems to decide the verdict. They are different questions. @@ -224,18 +227,6 @@ def json_type_of(cls: type[MetadataEntity]) -> object: return ZarrV3MetadataFieldJSON -def _json_shape(json_type: object) -> tuple[bool, bool]: - """Whether a JSON type admits a bare name, and whether it admits an object.""" - parts = get_args(json_type) if is_union(json_type) else (json_type,) - bare = any( - part is str - or (get_origin(part) is Literal and all(isinstance(v, str) for v in get_args(part))) - or getattr(part, "__supertype__", None) is str - for part in parts - ) - return bare, any(is_typeddict(part) for part in parts) - - def _is_entity_or_opaque(candidates: Sequence[object]) -> bool: """A nested metadata field: some entity kind, optionally with `Opaque`.""" return ( @@ -465,7 +456,7 @@ class Opaque: # have to spell it `super(Cls, self)`. CPython fixed this in 3.13, so when # that is the floor this is worth revisiting; the memory saved is small at # document scale, which is why it has not been. -_DERIVED: Final = ("member_types", "configuration_required", "nested_members") +_DERIVED: Final = ("member_types", "configuration_required", "nested_members", "name_members") """The class variables `_compile_entity` derives; a declaration of one is refused.""" @@ -485,13 +476,17 @@ def _compile_entity(cls: type[MetadataEntity]) -> None: raise TypeError(msg) cls.member_types, unread = derive_member_types(cls) if len(unread) != 0: + hints = field_hints(cls) msg = ( - f"{cls.__name__}: no check can be read off the annotation of " - f"{', '.join(sorted(unread))}; a field is one of the shapes JSON takes, " - "with any finer rule in `__post_init__`" + f"{cls.__name__}: {'; '.join(f'{name} is annotated {hints[name]!r}' for name in sorted(unread))}" + ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, a " + "Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass record, " + "Mapping[str, V], a NewType, or an entity kind with Opaque (CodecEntity | Opaque); " + "add | UNSET for an optional member, and put any finer rule in `__post_init__`" ) raise TypeError(msg) cls.configuration_required = any(required for required, _ in cls.member_types.values()) + cls.name_members = envelope_members(cls) hints = field_hints(cls) cls.nested_members = { name: annotation for name, annotation in hints.items() if contains_entity(annotation) @@ -520,28 +515,6 @@ def _final_methods_are_not_overridden(cls: type[MetadataEntity]) -> str | None: return None -def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | None: - # The named type is a promise about what `to_json` writes, and its - # shape follows from the members: a bare name only when no member is - # required and the entity must be understood, an object whenever - # there is a member to write or the flag to. - json_type = json_type_of(cls) - if json_type is ZarrV3MetadataFieldJSON: - return None - admits_bare, admits_object = _json_shape(json_type) - writes_bare = not cls.configuration_required and cls.must_understand - writes_object = len(cls.member_types) != 0 or not cls.must_understand - if admits_bare == writes_bare and admits_object == writes_object: - return None - return ( - f"{cls.__name__} names {json_type!r} as its JSON type, which " - f"{'admits' if admits_bare else 'lacks'} a bare name and " - f"{'admits' if admits_object else 'lacks'} an object, but the entity " - f"{'writes' if writes_bare else 'never writes'} a bare name and " - f"{'writes' if writes_object else 'never writes'} an object" - ) - - def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: # `MetadataEntity` itself is registered at no single point, so a # field typed as one could not be resolved through a scope. @@ -559,6 +532,43 @@ def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: ) +def _entity_unions_lacking_opaque(annotation: object) -> bool: + """Whether an entity kind appears in `annotation` without `Opaque` beside it.""" + inner, _ = strip_annotation(annotation) + if is_union(inner): + parts = [part for part in get_args(inner) if part is not UNSET] + if any(_is_entity_type(part) and part is not Opaque for part in parts): + return Opaque not in parts + return any(_entity_unions_lacking_opaque(part) for part in parts) + if _is_entity_type(inner): + return inner is not Opaque + if get_origin(inner) is tuple: + return any( + _entity_unions_lacking_opaque(part) for part in get_args(inner) if part is not Ellipsis + ) + if isinstance(inner, type) and is_dataclass(inner): + return any(_entity_unions_lacking_opaque(value) for value in field_hints(inner).values()) + return False + + +def _nested_fields_admit_opaque(cls: type[MetadataEntity]) -> str | None: + # A nested field holds an `Opaque` when the name is out of scope, so + # an annotation that excludes it lies to the type checker: reading + # `codec.inner.level` would be accepted and then raise. + lacking = sorted( + name + for name, annotation in cls.nested_members.items() + if _entity_unions_lacking_opaque(annotation) + ) + if len(lacking) == 0: + return None + return ( + f"{cls.__name__}: {', '.join(lacking)} holds an entity but does not admit Opaque, " + "which is what it holds when the name is out of scope; annotate it as the entity " + "kind | Opaque" + ) + + def _fields_do_not_shadow_class_variables(cls: type[MetadataEntity]) -> str | None: # A field of that name would go into `member_types`, into the # configuration, and into the JSON -- while the class variable it @@ -588,6 +598,163 @@ def _owed_class_variables_are_declared(cls: type[MetadataEntity]) -> str | None: return f"{cls.__name__} does not declare {', '.join(sorted(missing))}" +def _literal_class_variables_hold_a_listed_value(cls: type[MetadataEntity]) -> str | None: + # A class variable typed as a `Literal` -- `kind`, `scalar_storage` -- + # is read by other entities' rules, which have nothing to say about a + # value outside the listed ones and would fall silent. + for name, annotating in declared_class_vars(cls).items(): + if not hasattr(cls, name): + continue + shell = type( + "_ClassVar", + (), + { + "__annotations__": {name: own_annotations(annotating)[name]}, + "__module__": annotating.__module__, + }, + ) + try: + hint = get_type_hints(shell)[name] + except NameError: + # Typed with a name imported only for the type checker. + continue + inner = get_args(hint)[0] if get_origin(hint) is ClassVar else hint + if get_origin(inner) is Literal and getattr(cls, name) not in get_args(inner): + return ( + f"{cls.__name__} sets {name} = {getattr(cls, name)!r}, " + f"which is not one of {get_args(inner)!r}" + ) + return None + + +def _array_array_codecs_define_transition(cls: type[MetadataEntity]) -> str | None: + # The default `transition` is None -- undeterminable -- which stops + # every rule after the codec. Right for a codec that could not say; + # wrong to accept silently from one being written now. + if not (issubclass(cls, CodecEntity) and cls.kind == "array_array"): + return None + if cls.transition is not CodecEntity.transition: + return None + return ( + f"{cls.__name__} is an array_array codec and does not define transition; return " + "incoming if it leaves the array's parts unchanged, or the parts it hands the next codec" + ) + + +def _is_name_type(part: object) -> bool: + """A JSON type for the bare-name spelling: `str`, a `Literal` of names, or a `NewType` of `str`.""" + return ( + part is str + or (get_origin(part) is Literal and all(isinstance(v, str) for v in get_args(part))) + or getattr(part, "__supertype__", None) is str + ) + + +def _unaccepted(cls: type[MetadataEntity], name_type: object) -> list[str]: + """The names a `Literal` name type lists that the entity does not accept.""" + if get_origin(name_type) is not Literal: + return [] + return [value for value in get_args(name_type) if not cls.accepts(value)] + + +def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | None: + # The named type is a promise about what `to_json` writes, held to + # key by key: the spellings it admits are the ones the members make + # the entity write, its names are ones the entity accepts, and its + # configuration keys are the members. Value types are not compared + # -- a nested entity's JSON type and its field type are different + # spellings of one thing -- so that much stays with the tests. + json_type = json_type_of(cls) + if json_type is ZarrV3MetadataFieldJSON: + return None + parts = get_args(json_type) if is_union(json_type) else (json_type,) + objects = [part for part in parts if is_typeddict(part)] + names = [part for part in parts if _is_name_type(part)] + if len(objects) > 1 or len(names) > 1 or len(objects) + len(names) != len(parts): + return ( + f"{cls.__name__} names {json_type!r} as its JSON type, which is not an object " + "TypedDict, a name type, or a union of one of each" + ) + writes_bare = not cls.configuration_required and cls.must_understand + writes_object = len(cls.member_types) != 0 or not cls.must_understand + found: list[str] = [] + if writes_bare and len(names) == 0: + found.append("lacks the bare name the entity writes when every member is absent") + if not writes_bare and len(names) != 0: + found.append("admits a bare name, which the entity never writes") + if writes_object and len(objects) == 0: + found.append("lacks the object the entity writes") + if not writes_object and len(objects) != 0: + found.append("admits an object, which the entity never writes") + found.extend( + f"lists the name(s) {', '.join(map(repr, unaccepted))}, which the entity does not accept" + for name_type in names + if len(unaccepted := _unaccepted(cls, name_type)) != 0 + ) + for obj in objects: + try: + resolved = get_type_hints(obj, include_extras=True) + except NameError: + # Declared inside a function under postponed annotations: the + # names it uses are not reachable, so its keys go unjudged. + continue + hints = {key: strip_annotation(value)[0] for key, value in resolved.items()} + required: frozenset[str] = getattr(obj, "__required_keys__", frozenset()) + extra = sorted(hints.keys() - {"name", "configuration", "must_understand"}) + if len(extra) != 0: + found.append(f"has the key(s) {', '.join(extra)}, which no envelope has") + if "name" not in hints: + found.append("has no name key") + elif len(unaccepted := _unaccepted(cls, hints["name"])) != 0: + found.append( + f"names {', '.join(map(repr, unaccepted))}, which the entity does not accept" + ) + if not cls.must_understand and "must_understand" not in hints: + found.append("has no must_understand key, which the entity writes") + if len(cls.member_types) == 0: + if "configuration" in hints: + found.append("has a configuration key, and the entity has no members") + continue + if "configuration" not in hints: + found.append("has no configuration key, and the entity has members") + continue + if ("configuration" in required) != cls.configuration_required: + found.append( + "has configuration " + + ("required" if "configuration" in required else "optional") + + ", but a member is " + + ("required" if cls.configuration_required else "not required") + ) + configuration = hints["configuration"] + if not is_typeddict(configuration): + continue + try: + keys = get_type_hints(configuration, include_extras=True).keys() + except NameError: + continue + if set(keys) != set(cls.member_types): + found.append( + f"has configuration keys {sorted(keys)!r} where the members are " + f"{sorted(cls.member_types)!r}" + ) + continue + configuration_required: frozenset[str] = getattr( + configuration, "__required_keys__", frozenset() + ) + misstated = sorted( + key + for key, (member_required, _) in cls.member_types.items() + if (key in configuration_required) != member_required + ) + if len(misstated) != 0: + found.append( + f"states {', '.join(misstated)} with a requiredness the field does not give it" + ) + if len(found) == 0: + return None + return f"{cls.__name__} names {json_type!r} as its JSON type, which " + "; ".join(found) + + def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: """Each member's declared default, or `_MISSING_DEFAULT`. @@ -648,10 +815,13 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: _INVARIANTS: Final[tuple[Callable[[type[MetadataEntity]], str | None], ...]] = ( _final_methods_are_not_overridden, - _named_json_type_matches_what_is_written, _nested_kinds_have_a_point, + _nested_fields_admit_opaque, _fields_do_not_shadow_class_variables, _owed_class_variables_are_declared, + _named_json_type_matches_what_is_written, + _literal_class_variables_hold_a_listed_value, + _array_array_codecs_define_transition, _optional_members_default_to_unset, _required_members_have_no_default, ) @@ -663,8 +833,9 @@ class MetadataEntity(Generic[JSONT_co]): """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what - makes them well-typed by construction: an instance exists only if - `coerce` accepted the metadata that produced it. An optional member is + makes them well-typed when read: `coerce` builds one only from + metadata it accepted. Built by hand, the types are the caller's + promise -- `__post_init__` judges values, not types. An optional member is typed `| UNSET` with a default of `UNSET`, so absence is representable -- and distinct from a `null` the document wrote -- and a canonical spelling can leave it out. @@ -732,6 +903,14 @@ class creation: nothing declares them. JSON TypedDict is held to the same keys by `tests/v3/test_entities.py`. """ + name_members: ClassVar[tuple[str, ...]] = () + """The fields the envelope's name carries, marked `Annotated[str, FROM_NAME]`. + + Read off the fields at class creation. `coerce` fills each with the + name the document wrote, so a family whose validity is in its name + -- the raw-bytes `r` types -- needs no reading of its own. + """ + nested_members: ClassVar[Mapping[str, object]] = MappingProxyType({}) """The fields that hold other entities, with their annotations. @@ -765,6 +944,10 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: super().__init_subclass__(**kwargs) if base: return + if "__dataclass_fields__" in vars(cls): + # `@dataclass(slots=True)` builds the class a second time from + # the first one's dict, tables included: compiled already. + return _compile_entity(cls) for invariant in _INVARIANTS: message = invariant(cls) @@ -798,21 +981,25 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: "missing_key", ) configuration = cast("Mapping[str, object]", {}) - members, found, unreadable = coerce_members(configuration, cls.member_types) - if len(unreadable) == 0: - # Before judging: a member that is itself an entity has to be - # one before its container's value rules can ask it anything. - for name, annotation in cls.nested_members.items(): - if name in members: - members[name], nested = _resolve( - annotation, members[name], context, ("configuration", name) - ) - found = (*found, *nested) - if len(unreadable) != 0: - # A member that could not be read leaves a hole, and the value - # rules are written over a whole configuration -- blosc's - # `typesize` requirement reads `shuffle`. Judging around the - # hole would be guessing, so the type problems stand alone. + members, own = coerce_members(configuration, cls.member_types) + for member in cls.name_members: + members[member] = name + # A member that is itself an entity is read in the scope whatever + # else was found: its problems are determinable, so they are + # reported in the same pass. + found = own + for member, annotation in cls.nested_members.items(): + if member in members: + members[member], nested = _resolve( + annotation, members[member], context, ("configuration", member) + ) + found = (*found, *nested) + if any(entry.kind != "unknown_key" for entry in own): + # One of this entity's own members could not be read. That + # leaves a hole, and the rules are written over a whole + # configuration -- blosc's `typesize` requirement reads + # `shuffle` -- so judging around it would be guessing: the + # entity is not built, and the type problems stand alone. return None, found try: entity = cls(**members) @@ -821,6 +1008,11 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # rather than raised, located under the configuration. return None, (*found, *within((), refused.problems)) if any(entry.kind != "unknown_key" for entry in found): + # A contained entity could not be read. This entity's own + # rules ran -- an invalid inner is an `Opaque`, as an + # out-of-scope one is -- but what is handed back is not an + # entity that would be asked composition questions it cannot + # answer. return None, found return entity, found diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 604557df5f..21a343d207 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -30,6 +30,7 @@ ValidationProblem, validate_metadata_field_v3, ) +from zarr_metadata.v3._compile import is_class_var, own_annotations from zarr_metadata.v3._entity import ( CHUNK_GRID, CHUNK_KEY_ENCODING, @@ -184,6 +185,17 @@ def __post_init__(self) -> None: f"but its identifier is {entity.identifier!r}" ) raise ValueError(msg) + if "__dataclass_fields__" not in vars(entity) and any( + not is_class_var(annotation) for annotation in own_annotations(entity).values() + ): + # Class creation runs before `@dataclass` and cannot + # see whether it was applied; this is the next place + # the entity passes through before `coerce` builds it. + msg = ( + f"{entity.__name__} declares fields but is not a dataclass; decorate " + "it with @dataclass(frozen=True), which is what `coerce` builds it with" + ) + raise TypeError(msg) def extended_with(self, **entities: Unpack[PartialEntityTables]) -> Context: """This scope, plus entities of your own at the points named. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 628d833a15..6750c912c6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -10,16 +10,14 @@ import re from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, NewType, Self +from typing import Annotated, ClassVar, Final, NewType from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( FROM_NAME, - Coerced, DataTypeEntity, Loc, StorageClass, - named_configuration, problem, ) from zarr_metadata.v3.data_type._families import byte_values @@ -127,26 +125,6 @@ def accepts(cls, name: str) -> bool: """ return RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None - @classmethod - def coerce(cls, value: object, context: object) -> Coerced[Self]: - name, configuration, _ = named_configuration(value) - if name is None or not cls.accepts(name): - return None, problem((), "expected an 'r' raw-bytes data type") - found: tuple[ValidationProblem, ...] = () - if configuration is not None and len(configuration) != 0: - # Survivable, as an unknown key is everywhere else: the name - # still says everything this type is, so it is still read and - # its fill values are still judged. Returning nothing here let - # a stray key hide every other problem in the document. - found = problem(("configuration",), "'r' takes no configuration", "unknown_key") - try: - entity = cls(data_type_name=name) - except MetadataValidationError as refused: - return None, (*found, *refused.problems) - if any(entry.kind != "unknown_key" for entry in found): - return None, found - return entity, found - def __post_init__(self) -> None: """This family's validity is in its name, not in a configuration.""" found = _name_problems(self.data_type_name) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 9ce79dfd26..194ef41d14 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,22 +11,15 @@ import copy import dataclasses -import types from typing import ( Any, ClassVar, - NotRequired, Self, - Union, cast, - get_args, - get_origin, - get_type_hints, ) import pytest from hypothesis import given, settings -from typing_extensions import ReadOnly, is_typeddict from tests.rules.strategies import valid_documents from zarr_metadata.model import UNSET, MetadataValidationError @@ -123,36 +116,6 @@ } -@pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) -def test_the_constructor_mirrors_the_configuration(entity: type[MetadataEntity]) -> None: - # The member table and `configuration_required` are read off the fields, - # and the JSON type is named as the base's argument; the fields are the - # only spelling left that can drift from the public TypedDict -- and a - # field the TypedDict does not have would be a member no document could - # write. `must_understand` belongs to the object, not the configuration, - # so it is the one field the two deliberately do not share. - fields = {field.name for field in dataclasses.fields(entity)} - {"must_understand"} - json_type = json_type_of(entity) - objects = [part for part in _parts(json_type) if is_typeddict(part)] - if len(objects) == 0: - # A bare-name type: nothing to configure. `r` keeps its width - # in its name, so it holds a member that is not a configuration key. - assert fields == ({"data_type_name"} if entity is RawBytesDataType else set()) - return - (obj,) = objects - configuration = get_type_hints(obj, include_extras=True).get("configuration") - assert configuration is not None, f"{obj!r} has no configuration member" - while get_origin(configuration) in (NotRequired, ReadOnly): - (configuration,) = get_args(configuration) - assert fields == set(get_type_hints(configuration)) - - -def _parts(json_type: object) -> tuple[object, ...]: - return ( - get_args(json_type) if get_origin(json_type) in (Union, types.UnionType) else (json_type,) - ) - - @pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) def test_every_entity_names_its_json_type(entity: type[MetadataEntity]) -> None: # The default is not wrong, only uninformative; every entity this @@ -476,6 +439,46 @@ def test_an_unreadable_member_costs_the_entity() -> None: } +def test_an_optional_member_of_the_wrong_type_is_not_judged_as_absent() -> None: + # `typesize` could not be read. Building the entity around the hole + # would have `__post_init__` see it as absent and add a second, + # contradictory problem at the same location. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (4,)}}, + "chunk_key_encoding": "default", + "codecs": ( + "bytes", + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "typesize": "four", + "blocksize": 0, + }, + }, + ), + } + problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("codecs", 1, "configuration", "typesize"), "invalid_type") + ] + + +def test_the_document_writes_back_only_the_fields_it_read() -> None: + # An absent field is read as an `Opaque` standing in for it; writing + # it back as `null` would invent a value the document never wrote. + array, problems = read_array_v3({"shape": (4,)}, CORE_AND_EXTENSIONS) + assert problems == () + assert array.to_json() == {"shape": (4,)} + + def test_an_unreadable_member_is_not_judged_by_its_default() -> None: # `shuffle` could not be read, so it falls back to `noshuffle`, under # which `typesize` means nothing. The absent `typesize` must not be diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 5877610701..31ee49916f 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,7 +8,7 @@ import re from dataclasses import dataclass, replace -from typing import ClassVar, Literal, NotRequired, Self, cast +from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast import pytest from typing_extensions import TypedDict @@ -21,23 +21,22 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import json_type_of from zarr_metadata.v3.codec.blosc import BloscCodec -from zarr_metadata.v3.codec.gzip import GzipCodec +from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecObject from zarr_metadata.v3.entity import ( CORE, CORE_AND_EXTENSIONS, + FROM_NAME, ArrayDocumentV3, ArrayParts, ChunkGridEntity, CodecEntity, CodecKind, - Coerced, Context, DataTypeEntity, IntegerDataType, MetadataEntity, Opaque, StorageClass, - named_configuration, problem, ) @@ -301,7 +300,7 @@ class Int24DataType(IntegerDataType): # pyright: ignore[reportUnusedClass] class AcmeFixedDataType(DataTypeEntity): """`acme.fixedN`, a fixed-width type for every N.""" - data_type_name: str + data_type_name: Annotated[str, FROM_NAME] identifier: ClassVar[str] = "acme.fixed" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -310,13 +309,6 @@ class AcmeFixedDataType(DataTypeEntity): def accepts(cls, name: str) -> bool: return ACME_FIXED_PATTERN.fullmatch(name) is not None - @classmethod - def coerce(cls, value: object, context: object) -> Coerced[Self]: - name, _, _ = named_configuration(value) - if name is None or not cls.accepts(name): - return None, problem((), "expected an 'acme.fixedN' data type") - return cls(data_type_name=name), () - def to_json(self) -> ZarrV3MetadataFieldJSON: return cast("ZarrV3MetadataFieldJSON", self.data_type_name) @@ -345,7 +337,7 @@ def test_error_a_member_needs_a_check_from_somewhere() -> None: # An annotation outside the shapes `check_for` compiles implies no # check, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. - with pytest.raises(TypeError, match="annotation of inner; a field is one of the shapes JSON"): + with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): @dataclass(frozen=True) class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] @@ -492,6 +484,28 @@ class Vague(CodecEntity): # pyright: ignore[reportUnusedClass] kind: ClassVar[CodecKind] = "bytes_bytes" +# The JSON types third-party entities name, at module level so their +# annotations resolve. +class AcmeLvlConfiguration(TypedDict, closed=True): + lvl: int + + +class AcmeLvlObject(TypedDict, closed=True): + name: Literal["acme.lvl"] + configuration: AcmeLvlConfiguration + must_understand: NotRequired[bool] + + +class AcmeBlockConfiguration(TypedDict, closed=True): + block: int + + +class AcmeBlockObject(TypedDict, closed=True): + name: Literal["acme.block"] + configuration: AcmeBlockConfiguration + must_understand: NotRequired[bool] + + # A third-party rule about a member, written in `__post_init__`. @dataclass(frozen=True) class AcmeBlockCodec(CodecEntity): @@ -530,10 +544,123 @@ def test_a_rule_about_a_member_is_post_init() -> None: assert [p.kind for p in problems] == ["invalid_type"] +def test_a_slotted_entity_is_compiled_once() -> None: + # `@dataclass(slots=True)` builds the class twice; the second pass + # arrives with the derived tables already on it and must not be + # refused as having declared them. + @dataclass(frozen=True, slots=True) + class AcmeSlotted(CodecEntity): + level: int + + identifier: ClassVar[str] = "acme.slotted" + kind: ClassVar[CodecKind] = "bytes_bytes" + + assert list(AcmeSlotted.member_types) == ["level"] + assert AcmeSlotted(level=1).to_json() == { + "name": "acme.slotted", + "configuration": {"level": 1}, + } + + +def test_a_bare_class_var_is_a_class_variable() -> None: + @dataclass(frozen=True) + class AcmeNoted(CodecEntity): + identifier: ClassVar[str] = "acme.noted" + kind: ClassVar[CodecKind] = "bytes_bytes" + note: ClassVar = "not a member" + + assert AcmeNoted.member_types == {} + + +def test_a_number_member_is_a_float_field() -> None: + # JSON has one number type; `float` admits an int spelled without a + # point and refuses a bool, which is what a document's `2` and `true` + # deserve. + @dataclass(frozen=True) + class AcmeScaled(CodecEntity): + scale: float + + identifier: ClassVar[str] = "acme.scaled" + kind: ClassVar[CodecKind] = "array_array" + + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + return incoming + + scope = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeScaled.identifier: AcmeScaled}) + for written in (2, 2.5): + codec, problems = scope.coerce( + "codecs", {"name": "acme.scaled", "configuration": {"scale": written}} + ) + assert problems == () + assert isinstance(codec, AcmeScaled) + _, problems = scope.coerce("codecs", {"name": "acme.scaled", "configuration": {"scale": True}}) + assert [(p.loc, p.message) for p in problems] == [ + (("configuration", "scale"), "expected a number, got True") + ] + + +def test_error_an_entity_must_be_a_dataclass() -> None: + # Class creation runs before `@dataclass` and cannot see it missing; + # registration can, and says so instead of the first `coerce` failing + # with the base class's `__init__`. + class Undecorated(CodecEntity): + level: int + + identifier: ClassVar[str] = "acme.undecorated" + kind: ClassVar[CodecKind] = "bytes_bytes" + + with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): + CORE_AND_EXTENSIONS.extended_with(codecs={Undecorated.identifier: Undecorated}) + + +def test_error_a_nested_field_admits_opaque() -> None: + # What the field holds when the inner name is out of scope. + with pytest.raises(TypeError, match="inner holds an entity but does not admit Opaque"): + + @dataclass(frozen=True) + class Closed(CodecEntity): # pyright: ignore[reportUnusedClass] + inner: CodecEntity + + identifier: ClassVar[str] = "acme.closed" + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_error_an_array_array_codec_defines_transition() -> None: + # Left at the default, every rule after the codec would go silent. + with pytest.raises(TypeError, match="array_array codec and does not define transition"): + + @dataclass(frozen=True) + class Silent(CodecEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.silent" + kind: ClassVar[CodecKind] = "array_array" + + +def test_error_a_literal_class_variable_holds_a_listed_value() -> None: + # `bytes` asks a data type's storage class and has nothing to say + # about a fourth value: the endian rule would silently not apply. + with pytest.raises( + TypeError, match="sets scalar_storage = 'sixteen_bytes', which is not one of" + ): + + @dataclass(frozen=True) + class Wide(DataTypeEntity): # pyright: ignore[reportUnusedClass] + identifier: ClassVar[str] = "acme.wide" + scalar_storage: ClassVar[StorageClass] = "sixteen_bytes" # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + + +def test_error_a_list_of_problem_tuples_is_refused() -> None: + # `problem()` returns a one-element tuple; a list of those would pass + # the constructor and fail inside `coerce`, far from the mistake. + with pytest.raises(TypeError, match="collect with `extend`, not `append`"): + MetadataValidationError([problem(("a",), "bad a")]) # type: ignore[list-item] # pyright: ignore[reportArgumentType] + + def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: # A required member means the entity is always written as an object, # so naming a bare-name type for it is a promise `to_json` would break. - with pytest.raises(TypeError, match="lacks an object, but the entity never writes a bare name"): + with pytest.raises( + TypeError, match="admits a bare name, which the entity never writes; lacks the object" + ): @dataclass(frozen=True) class Misnamed(CodecEntity[Literal["acme.misnamed"]]): # pyright: ignore[reportUnusedClass] @@ -543,19 +670,37 @@ class Misnamed(CodecEntity[Literal["acme.misnamed"]]): # pyright: ignore[report kind: ClassVar[CodecKind] = "bytes_bytes" +def test_error_the_named_json_type_must_name_what_the_entity_accepts() -> None: + # `GzipCodecObject` spells `name: Literal["gzip"]`; an entity that + # accepts only its own name cannot write that. + with pytest.raises(TypeError, match="names 'gzip', which the entity does not accept"): + + @dataclass(frozen=True) + class Impostor(CodecEntity[GzipCodecObject]): # pyright: ignore[reportUnusedClass] + level: int + + identifier: ClassVar[str] = "acme.impostor" + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_error_the_named_json_type_must_have_the_members_as_keys() -> None: + with pytest.raises( + TypeError, match=r"configuration keys \['lvl'\] where the members are \['level'\]" + ): + + @dataclass(frozen=True) + class Mismatched(CodecEntity[AcmeLvlObject]): # pyright: ignore[reportUnusedClass] + level: int + + identifier: ClassVar[str] = "acme.lvl" + kind: ClassVar[CodecKind] = "bytes_bytes" + + def test_a_third_party_entity_may_name_its_json_type_or_not() -> None: # Left defaulted, `to_json` is typed as any metadata field; named, as # the entity's own type -- and either way the same dict comes back. assert json_type_of(AcmeLz4Codec) is ZarrV3MetadataFieldJSON - class AcmeBlockConfiguration(TypedDict, closed=True): - block: int - - class AcmeBlockObject(TypedDict, closed=True): - name: Literal["acme.block"] - configuration: AcmeBlockConfiguration - must_understand: NotRequired[bool] - @dataclass(frozen=True) class AcmeTypedBlockCodec(CodecEntity[AcmeBlockObject]): block: int From 64d2c32db2a7bd9de9480ca9f3d3dc79d941f146 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 14:42:32 +0200 Subject: [PATCH 075/107] docs(zarr-metadata): the door is the guide its example claimed to be Four adversarial reviews -- two extension authors writing a codec and a data type against the door alone, a design review, an onboarding review -- agreed on one thing: the door's docstring documented a `bytes_bytes` codec with one `int` member, used five names it did not export, and said nothing about what an entity answers for itself. Now it is runnable as written and covers what comes back (three `loc` bases, the two `coerce` contracts), the shapes a field may take, the `problem()` tuple idiom, the composition contract per kind, the named JSON type, and which mistakes are refused. It exports what its example needs -- `UNSET`, `MetadataValidationError`, `ValidationProblem`, `ProblemKind`, `JSONValue`, `ZarrV3MetadataFieldJSON` -- and no longer the hand-written check vocabulary the compiler made obsolete (53 names to 41). Every class-creation and registration refusal now says what to write; `must_understand`'s summary no longer says the opposite of its name; `problem`'s says it returns a tuple. The extension test module imports public modules only, as its own docstring promised. The two reviewer- written extensions are kept as `tests/v3/test_acme_affine.py` and `tests/v3/test_acme_decimal.py`, the complete examples the door points at. The named JSON type's requiredness is read off resolved hints, since a TypedDict under postponed annotations cannot see its own `NotRequired`. The entity page hides inherited members; the API index lists it; three fragments no longer describe hooks that are gone. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 20 +- .../zarr-metadata/changes/4379.feature.md | 11 +- packages/zarr-metadata/changes/4379.misc.2.md | 20 + packages/zarr-metadata/docs/api/index.md | 4 + packages/zarr-metadata/docs/api/v3/entity.md | 3 + .../src/zarr_metadata/v3/_checks.py | 8 +- .../src/zarr_metadata/v3/_entity.py | 47 ++- .../src/zarr_metadata/v3/_registry.py | 5 +- .../src/zarr_metadata/v3/entity.py | 236 +++++++----- .../zarr-metadata/tests/test_public_api.py | 3 - .../tests/v3/test_acme_affine.py | 223 +++++++++++ .../tests/v3/test_acme_decimal.py | 354 ++++++++++++++++++ .../tests/v3/test_extension_api.py | 9 +- 13 files changed, 809 insertions(+), 134 deletions(-) create mode 100644 packages/zarr-metadata/tests/v3/test_acme_affine.py create mode 100644 packages/zarr-metadata/tests/v3/test_acme_decimal.py diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 9619572eab..ae106a9c68 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -35,18 +35,16 @@ alias, an array homogeneous or fixed, a union of those, a nested object described by a TypedDict or a record dataclass, an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested metadata field -- an entity type, with or without `Opaque`. That covers -every member this package models; an annotation the compiler does not -read is refused at class creation until `register_check` teaches it the -shape, and `Annotated[str, FROM_NAME]` marks the one field carried by -the envelope's name rather than a configuration key (`r`). `configuration_required` follows too, since -the spec ties the bare-name spelling to whether any member is required. -The public JSON TypedDict is no longer read by the package at all: the -fields are held to its keys by a test, which is the one correspondence -still written by hand. +every member this package models; an annotation outside them is refused +at class creation, and `Annotated[str, FROM_NAME]` marks the one field +carried by the envelope's name rather than a configuration key (`r`), +which `coerce` fills from the envelope. `configuration_required` follows +too, since the spec ties the bare-name spelling to whether any member is +required. The public JSON TypedDict an entity names as its base's +argument is held to the fields at class creation, key by key. -Three things a class may no longer restate, because the fields already -say them: `configuration_required`, the requiredness of a hand-written -entry, and a member with no check from either source. Field annotations +Nothing a class may restate: every table the layer reads is derived +from the fields, and declaring one is refused. Field annotations are resolved per class, skipping class variables by text, so a `ClassVar` naming something imported only for the type checker cannot fail class creation. diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index bb04fa9a61..98aa42cf9c 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -9,8 +9,10 @@ data type, chunk grid and chunk key encoding — is a class that answers for itself: - `coerce` reads raw metadata into the entity, or says why it is not one; -- `problems` says which of its own values the spec disallows; -- `to_json` gives its simplest equivalent spelling; +- `__post_init__` refuses the values the spec disallows, collecting every + problem and raising once; `coerce` reports the same problems instead; +- `to_json` writes it back as it was read, and `canonical` gives its + simplest equivalent spelling; - `incoming_problems`, `shape_problems`, `fill_value_problems` and `transition` answer the questions that need a piece of the document: the array reaching a codec, the shape a grid divides, the fill value a @@ -18,7 +20,8 @@ for itself: Validating a document is composing those answers. Nothing in the document layer knows what `blosc` or `int32` or `rectilinear` is, so -adding an extension is a class and one registry entry. +adding an extension is a class and a scope that includes it +(`CORE_AND_EXTENSIONS.extended_with(...)`, from `zarr_metadata.v3.entity`). - **What is judged**: fill value against data type (recursively, through `struct` fields and `cast_value` targets), codec pipeline kind @@ -31,7 +34,7 @@ adding an extension is a class and one registry entry. chunk they will be handed). v2 arrays get chunks/shape rank agreement; v3 groups recurse into inline consolidated metadata, judging each embedded child document at its path. -- **Scope**: `v3._registry` maps each extension point to the +- **Scope**: `zarr_metadata.v3.entity` maps each extension point to the identifiers in play, in two scopes — `CORE`, what the specification defines, and `CORE_AND_EXTENSIONS`, which adds what `zarr-extensions` registers and this package models. A name in neither is not rejected; diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index b8d0835034..f85efed366 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -48,6 +48,26 @@ names from earlier drafts of this branch, never released -- are gone, and declaring any derived class variable is refused, where before only `configuration_required` was. +Four adversarial reviews -- two extension authors writing a codec and a +data type against the door alone, a design review, an onboarding review +-- drove the last round. `coerce` no longer builds an entity around a +member that could not be read, which had `__post_init__` judging the +hole; nested entities are resolved whatever else was found, so their +problems come out in the same pass; `FROM_NAME` fields are filled by +the base; `to_json` on a document writes only the fields it had; `float` +is a shape. Class creation and registration refuse the mistakes both +authors made or nearly made -- no `@dataclass`, a nested field without +`Opaque`, an `array_array` codec without `transition`, a `Literal`-typed +class variable outside its values, a list of `problem()` tuples -- with +messages that say what to write. The door's docstring is now the guide +its example claimed to be, runnable as written, with the composition +contract per kind and what comes back; it exports what its example +needs (`UNSET`, `MetadataValidationError`, `ValidationProblem`, +`ProblemKind`, `JSONValue`, `ZarrV3MetadataFieldJSON`) and no longer the +hand-written check vocabulary the compiler made obsolete. The two +reviewer-written extensions are kept as `tests/v3/test_acme_affine.py` +and `tests/v3/test_acme_decimal.py`, written against the door alone. + One thing this does not change, under mypy. An entity's JSON type is a TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as `Mapping[str, object]`, never as the diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 7d42570f29..15026ff2c9 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -20,6 +20,10 @@ The package is organized to mirror the structure of the Zarr specifications: documents, with subpackages for [chunk grids](v3/chunk_grid.md), [chunk key encodings](v3/chunk_key_encoding.md), [codecs](v3/codec.md), and [data types](v3/data_type.md) +- [`zarr_metadata.v3.entity`](v3/entity.md) — the extension layer: read a + document into entities that answer for themselves, and write your own + codec, data type or chunk grid and add it to a scope; its module + docstring is the guide The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so diff --git a/packages/zarr-metadata/docs/api/v3/entity.md b/packages/zarr-metadata/docs/api/v3/entity.md index 9ce2e1c6dd..bd846b75c7 100644 --- a/packages/zarr-metadata/docs/api/v3/entity.md +++ b/packages/zarr-metadata/docs/api/v3/entity.md @@ -3,3 +3,6 @@ title: entity --- ::: zarr_metadata.v3.entity + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py index 73cc75322e..78a0bc6d06 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py @@ -48,7 +48,13 @@ def problem( loc: Loc, message: str, kind: ProblemKind = "invalid_type" ) -> tuple[ValidationProblem, ...]: - """One problem, as the tuple every check returns.""" + """One problem, as the one-element tuple every check returns. + + A tuple so that a check can return it directly and a rule can + `found.extend(problem(...))` and raise `MetadataValidationError(found)` + once. The default `kind` names a type mismatch; a value rule passes + `"invalid_value"`. + """ return (ValidationProblem(loc, message, kind),) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index cb4922152f..4ca5a769c9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -41,6 +41,7 @@ Final, Generic, Literal, + NotRequired, TypeAlias, cast, final, @@ -471,7 +472,7 @@ def _compile_entity(cls: type[MetadataEntity]) -> None: if len(declared) != 0: msg = ( f"{cls.__name__} declares {', '.join(declared)}, which is derived from the " - "fields at class creation" + "fields at class creation; remove the declaration" ) raise TypeError(msg) cls.member_types, unread = derive_member_types(cls) @@ -582,8 +583,8 @@ def _fields_do_not_shadow_class_variables(cls: type[MetadataEntity]) -> str | No if len(shadowed) == 0: return None return ( - f"{cls.__name__} declares {', '.join(shadowed)} as a field, " - "shadowing a class variable of the same name" + f"{cls.__name__} declares {', '.join(shadowed)} as a field, shadowing a class " + "variable of the same name; rename the field, or set the class variable instead" ) @@ -592,10 +593,15 @@ def _owed_class_variables_are_declared(cls: type[MetadataEntity]) -> str | None: # is one the concrete entity owes: `identifier` for all of them, # `kind` for a codec, `bounds` for an integer type. Derived rather # than listed, so adding one to a family cannot forget to require it. - missing = [name for name in declared_class_vars(cls) if not hasattr(cls, name)] + annotated = declared_class_vars(cls) + missing = sorted(name for name in annotated if not hasattr(cls, name)) if len(missing) == 0: return None - return f"{cls.__name__} does not declare {', '.join(sorted(missing))}" + owed = ", ".join(f"{name} (annotated by {annotated[name].__name__})" for name in missing) + return ( + f"{cls.__name__} does not declare {owed}; set each as a class variable, " + "or pass base=True if this class exists only to be subclassed" + ) def _literal_class_variables_hold_a_listed_value(cls: type[MetadataEntity]) -> str | None: @@ -699,7 +705,10 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | # names it uses are not reachable, so its keys go unjudged. continue hints = {key: strip_annotation(value)[0] for key, value in resolved.items()} - required: frozenset[str] = getattr(obj, "__required_keys__", frozenset()) + # Requiredness from the resolved hints, not `__required_keys__`: + # under postponed annotations a TypedDict's own metaclass cannot + # see `NotRequired` inside a string. + required = {key for key, value in resolved.items() if get_origin(value) is not NotRequired} extra = sorted(hints.keys() - {"name", "configuration", "must_understand"}) if len(extra) != 0: found.append(f"has the key(s) {', '.join(extra)}, which no envelope has") @@ -729,18 +738,21 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | if not is_typeddict(configuration): continue try: - keys = get_type_hints(configuration, include_extras=True).keys() + configuration_hints = get_type_hints(configuration, include_extras=True) except NameError: continue + keys = configuration_hints.keys() if set(keys) != set(cls.member_types): found.append( f"has configuration keys {sorted(keys)!r} where the members are " f"{sorted(cls.member_types)!r}" ) continue - configuration_required: frozenset[str] = getattr( - configuration, "__required_keys__", frozenset() - ) + configuration_required = { + key + for key, value in configuration_hints.items() + if get_origin(value) is not NotRequired + } misstated = sorted( key for key, (member_required, _) in cls.member_types.items() @@ -789,8 +801,10 @@ def _optional_members_default_to_unset(cls: type[MetadataEntity]) -> str | None: if len(invented) == 0: return None return ( - f"{cls.__name__} gives the optional member(s) " - f"{', '.join(invented)} a default other than UNSET" + f"{cls.__name__} gives the optional member(s) {', '.join(invented)} a default " + "other than UNSET; write `| UNSET = UNSET` and read the meaning of absence where " + "the member is used -- a default written into every document is not a member the " + "document left out" ) @@ -808,8 +822,8 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: if len(presumed) == 0: return None return ( - f"{cls.__name__} gives the required member(s) " - f"{', '.join(presumed)} a default; required members have none" + f"{cls.__name__} gives the required member(s) {', '.join(presumed)} a default; " + "either drop the default, or make the member optional with `| UNSET = UNSET`" ) @@ -868,7 +882,10 @@ class creation: nothing declares them. """ must_understand: ClassVar[bool] = True - """Whether a reader that does not know this entity may skip it. + """Whether a reader must understand this entity to read the array. + + `True`, the default and every codec: a reader that does not know it + may not skip it. A property of the *kind* of metadata, not of a use of it: a codec is something you must understand, every time it appears, because diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 21a343d207..1932215cef 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -174,15 +174,18 @@ def __post_init__(self) -> None: for field, table in self.tables().items(): for key, entity in table.items(): if not issubclass(entity, _ENTITY_KINDS[field]): + point = entity.extension_point msg = ( f"{entity.__name__} is registered at {field!r}, which takes " f"{_ENTITY_KINDS[field].__name__} entities" + + (f"; register it at {point!r}" if point is not None else "") ) raise TypeError(msg) if key != entity.identifier: msg = ( f"{entity.__name__} is registered at {field!r} under {key!r} " - f"but its identifier is {entity.identifier!r}" + f"but its identifier is {entity.identifier!r}; key the table by " + f"{entity.__name__}.identifier" ) raise ValueError(msg) if "__dataclass_fields__" not in vars(entity) and any( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 9d47aec43c..03f450f4d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -3,16 +3,16 @@ Every Zarr v3 extension point -- codecs, data types, chunk grids, chunk key encodings, storage transformers -- is modelled as a class that answers for itself. This module is the public door to that layer, for two -kinds of caller. +kinds of caller, and everything either needs is exported from it. **Reading metadata.** `ArrayDocumentV3.from_json` is the fail-fast front door: one call, and either every extension point is read or a single -`MetadataValidationError` carries every reason it is not. The entities it -yields know things the document does not spell out -- what a data type's -scalars are, which position a codec occupies, what a grid divides an -array into. A name the scope does not model is not a failure: it arrives -as an `Opaque` marked `out_of_scope`, for the reader to resolve -elsewhere. +`MetadataValidationError` carries every reason it is not, in +`.problems`. The entities it yields know things the document does not +spell out -- what a data type's scalars are, which position a codec +occupies, what a grid divides an array into. A name the scope does not +model is not a failure: it arrives as an `Opaque` marked `out_of_scope`, +for the reader to resolve elsewhere. from zarr_metadata.v3.entity import ArrayDocumentV3, CodecEntity @@ -24,73 +24,155 @@ else: codec.json, codec.reason # 'out_of_scope': resolve it yourself +**What comes back.** Problems, not exceptions, wherever a document is +being judged rather than demanded. `zarr_metadata.rules.validate_array_metadata_v3(document, context=...)` +returns a tuple of `ValidationProblem(loc, message, kind)`, `kind` one of +`ProblemKind`, each `loc` indexing into the document: +`("codecs", 1, "configuration", "level")`. `SCOPE.coerce("codecs", entry)` +reads one metadata field and returns `(entity, problems)` where `entity` +is the entity or an `Opaque` -- never `None` -- with `loc` relative to the +entry: `("configuration", "level")`. An entity's own +`coerce(value, context)` returns `(entity or None, problems)`; that is +`Coerced`. Constructing an entity by hand raises `MetadataValidationError` +with `loc` relative to the configuration: `("level",)`. + **Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, -`ChunkGridEntity` or `MetadataEntity`, declare the fields, and add it to -a scope. Name your JSON type as the base's argument if you have one -- -`CodecEntity[AcmeLz4Metadata]` -- and `to_json` is typed as it; left -bare, `to_json` is typed as any metadata field: +`ChunkGridEntity` or `MetadataEntity`; declare the configuration as +dataclass fields; put every rule finer than a type in `__post_init__`; +add the class to a scope. Complete, and runnable as written: + + from dataclasses import dataclass + from typing import ClassVar + + from zarr_metadata.rules import validate_array_metadata_v3 + from zarr_metadata.v3.entity import ( + CORE_AND_EXTENSIONS, + UNSET, + CodecEntity, + CodecKind, + MetadataValidationError, + problem, + ) - @dataclass(frozen=True) + @dataclass(frozen=True) # load-bearing: `coerce` builds the entity with cls(**members) class AcmeLz4Codec(CodecEntity): - # A required member has no default; an optional one defaults to - # UNSET, so absence stays distinct from a JSON null. - acceleration: int | UNSET = UNSET + acceleration: int | UNSET = UNSET # optional: defaults to UNSET, never to a value identifier: ClassVar[str] = "acme.lz4" kind: ClassVar[CodecKind] = "bytes_bytes" - SCOPE = CORE_AND_EXTENSIONS.extended_with( - codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}, - ) - + def __post_init__(self) -> None: + if self.acceleration is not UNSET and not 1 <= self.acceleration <= 65537: + raise MetadataValidationError( + problem( + ("acceleration",), + f"expected an integer in [1, 65537], got {self.acceleration}", + "invalid_value", + ) + ) + + SCOPE = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}) validate_array_metadata_v3(document, context=SCOPE) The fields are the only place the shape is written. Which members exist, which may be left out (the type admits `UNSET`), and how each one is -type-checked are all read off the annotations -- an `int`, a `Literal` -of names, an array, a nested object, a nested entity type: the shapes -JSON takes, and no others. Everything finer -- a bound, a rule about a -member, members read together -- is `__post_init__`, in plain code, -collecting every problem and raising once; `coerce` reports those -instead of raising, located under the configuration: - - acceleration: int | UNSET = UNSET - - def __post_init__(self) -> None: - if self.acceleration is not UNSET and not 1 <= self.acceleration <= 65537: - raise MetadataValidationError( - problem(("acceleration",), f"expected an integer in [1, 65537], got {self.acceleration}", "invalid_value") - ) +type-checked are all read off the annotations, and the shapes are the +ones JSON takes: `int`, `float` (any JSON number), `bool`, `str`, +`JSONValue`, a `Literal` of names, `tuple[T, ...]` or `tuple[T1, T2]`, a +TypedDict or dataclass record, `Mapping[str, V]`, a `NewType`, and a +nested entity -- always with `Opaque`, `inner: CodecEntity | Opaque`, +because that is what the field holds when the inner name is out of +scope. Anything else is refused at class creation. A required member +has no default; an optional one is `| UNSET = UNSET`, so absence stays +distinct from a JSON `null`, and a member that means something when +absent is read that way where it is used, not defaulted. + +Everything finer than a type -- a bound, a rule about one member, members +read together -- is `__post_init__`, in plain code. It collects every +problem it finds and raises once; `coerce` catches the same error and +reports the problems in the document instead. `problem(loc, message, +kind)` returns a *one-element tuple*, so several are collected with +`found.extend(problem(...))` and raised as `MetadataValidationError(found)`; +pass `kind="invalid_value"` for a value rule, since the default names a +type mismatch. `__post_init__` runs only on an entity whose members all +read: a member of the wrong type is reported and the entity is not built. + +**What an entity answers for itself**, beyond its fields. `to_json`, +`canonical` and `coerce` are written once in the base; an entity whose +own members have two spellings that mean the same overrides +`simplified`, which `canonical` calls (overriding `canonical` itself is +refused). Then, by kind: + +- Every entity: `identifier`, the name it is registered under. A family + -- one class for every `acme.fixedN` -- overrides `accepts(name)` and + keeps the name in a field marked `Annotated[str, FROM_NAME]`, which + `coerce` fills from the envelope. +- A codec: `kind`. An `array_array` codec must define + `transition(incoming: ArrayParts) -> ArrayParts | None` -- return + `incoming` if it leaves the array's shape, grid and data type alone, + or the parts it hands the next codec -- and may define + `incoming_problems(incoming)` for what it cannot take. `variable_size` + says its output length is not fixed. A `bytes_bytes` or `array_bytes` + codec defines neither. +- A data type: `scalar_storage`, one of `StorageClass` (the `bytes` + codec asks it whether an endianness is needed), and + `fill_value_problems(value, loc)`, which judges a document's + `fill_value`; left undefined, every fill value is accepted. The + families `IntegerDataType`, `FloatDataType`, `ComplexDataType` and + `NumpyTimeDataType` carry those for the types they cover; a family of + your own is a subclass declared with `base=True`, which owes nothing + itself and passes its class variables down. +- A chunk grid: `grid(array_shape)` and `shape_problems`; see + `ChunkGridEntity`. + +The defaults fail closed for the package's own sake, so the ones an +author would otherwise miss -- an `array_array` codec without a +`transition`, a data type with a `scalar_storage` outside the listed +values, a nested field without `Opaque`, a class without `@dataclass` +(caught at registration, the first place that can see it) -- are refused +with a message that says what to write. + +**Naming the JSON type.** `CodecEntity[AcmeLz4Metadata]` types `to_json` +as your own TypedDict rather than as any metadata field. The shape is +`{name: Literal["acme.lz4"], configuration: AcmeLz4Configuration, +must_understand: NotRequired[bool]}`, in a union with the name literal +only if no member is required; class creation holds it to the entity +key by key (names it accepts, the members as configuration keys with +the members' requiredness), and the tests hold the value types. + +Two complete extensions written against this module alone, as tests: +`tests/v3/test_acme_affine.py` (an `array_array` codec with a number, an +optional member and a nested data type) and +`tests/v3/test_acme_decimal.py` (a configured data type with a +fill-value rule). A name in no scope is not rejected -- that is what extension openness means -- so registering yours is how you get it judged rather than waved -through. +through. `CORE` is what the specification defines; `CORE_AND_EXTENSIONS` +adds the `zarr-extensions` registry; `extended_with` adds yours. -One known friction, under mypy only. An entity's `to_json` returns its own -object TypedDict, and mypy does not accept that where a +One known friction, under mypy only. An entity's `to_json` returns its +own object TypedDict, and mypy does not accept that where a `ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as `Mapping[str, object]`, never as the `Mapping[str, JSONValue]` the -envelope declares. So putting `to_json()` output straight into a `codecs` -list needs a `cast` under mypy. Pyright accepts it. - -That conversion is sound here, which is why pyright is the one that is -right. The rule mypy is applying exists because an ordinary TypedDict may -carry extra items of types it never declared, so the union of the -declared value types does not bound what is in the mapping. Every -TypedDict in this package is `closed` (PEP 728), which forbids exactly -that, and pyright implements PEP 728. Mypy does not yet -- see -python/mypy#8994 and python/mypy#18439. - -The type therefore stays as it is. Widening `configuration` to -`Mapping[str, object]` or `Mapping[str, Any]` would satisfy mypy by -making the annotation say something false: a configuration's values are -JSON, and that is worth more than one checker's `cast`. +envelope declares (python/mypy#8994, python/mypy#18439 -- mypy lacks +PEP 728, which every TypedDict here relies on). The conversion is sound +and the annotation stays; a consumer under mypy casts at the one place +it puts an entity's JSON into a document. Pyright accepts it. """ from __future__ import annotations -from zarr_metadata.v3._chain import chain_problems, order_problems -from zarr_metadata.v3._document import ArrayDocumentV3, array_problems_v3, read_array_v3 +from zarr_metadata._common import JSONValue +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + MetadataValidationError, + ProblemKind, + ValidationProblem, +) +from zarr_metadata.v3._chain import chain_problems +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3._document import ArrayDocumentV3 from zarr_metadata.v3._entity import ( CHUNK_GRID, CHUNK_KEY_ENCODING, @@ -105,40 +187,21 @@ def __post_init__(self) -> None: DataTypeEntity, ExtensionPointField, Loc, - MemberTypes, MetadataEntity, Opaque, StorageClass, - TypeCheck, - coerce_members, - is_bool, - is_int, is_integer, - is_json_value, - is_metadata_field, - is_str, named_configuration, - one_of, problem, - sequence_of, within, ) -from zarr_metadata.v3._parts import UNKNOWN_GRID, ArrayParts, ChunkGrid, Extents, shard_index_grid -from zarr_metadata.v3._registry import ( - CORE, - CORE_AND_EXTENSIONS, - Context, - EntityTables, - PartialEntityTables, -) +from zarr_metadata.v3._parts import ArrayParts, ChunkGrid, Extents +from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context, EntityTables from zarr_metadata.v3.data_type._families import ( - FLOAT_SPECIALS, ComplexDataType, FloatDataType, IntegerDataType, NumpyTimeDataType, - as_sequence, - byte_values, ) __all__ = [ @@ -148,10 +211,9 @@ def __post_init__(self) -> None: "CORE", "CORE_AND_EXTENSIONS", "DATA_TYPE", - "FLOAT_SPECIALS", "FROM_NAME", "STORAGE_TRANSFORMERS", - "UNKNOWN_GRID", + "UNSET", "ArrayDocumentV3", "ArrayParts", "ChunkGrid", @@ -167,31 +229,19 @@ def __post_init__(self) -> None: "Extents", "FloatDataType", "IntegerDataType", + "JSONValue", "Loc", - "MemberTypes", "MetadataEntity", + "MetadataValidationError", "NumpyTimeDataType", "Opaque", - "PartialEntityTables", + "ProblemKind", "StorageClass", - "TypeCheck", - "array_problems_v3", - "as_sequence", - "byte_values", + "ValidationProblem", + "ZarrV3MetadataFieldJSON", "chain_problems", - "coerce_members", - "is_bool", - "is_int", "is_integer", - "is_json_value", - "is_metadata_field", - "is_str", "named_configuration", - "one_of", - "order_problems", "problem", - "read_array_v3", - "sequence_of", - "shard_index_grid", "within", ] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index b54713ba61..ba037658da 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -291,9 +291,7 @@ def test_all_is_grouped_and_unique() -> None: "CastOutOfRangeMode", "CastRoundingMode", "CodecKind", - "TypeCheck", "StorageClass", - "MemberTypes", "Loc", "Extents", "ExtensionPointField", @@ -311,7 +309,6 @@ def test_all_is_grouped_and_unique() -> None: "JSONValue", "MetadataValidationError", "Opaque", - "PartialEntityTables", "NumpyDatetime64", "NumpyTimeUnit", "NumpyTimedelta64", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py new file mode 100644 index 0000000000..22eb71cd17 --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -0,0 +1,223 @@ +"""`acme.affine`, a third-party `array_array` codec, written against the door alone. + +Every element becomes `x * scale + offset`, and the result may be stored +as a different data type. Every import is from `zarr_metadata.v3.entity` +or another public module; this is the complete example the door's +docstring points at. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, Self, cast + +import pytest +from typing_extensions import TypedDict + +from zarr_metadata.rules import ( + Canonical, + canonicalize_array_metadata_v3, + validate_array_metadata_v3, +) +from zarr_metadata.v3.data_type.float32 import Float32DataType +from zarr_metadata.v3.entity import ( + CORE_AND_EXTENSIONS, + UNSET, + ArrayDocumentV3, + ArrayParts, + CodecEntity, + CodecKind, + DataTypeEntity, + MetadataValidationError, + Opaque, + ValidationProblem, + ZarrV3MetadataFieldJSON, + problem, +) + +if TYPE_CHECKING: + from zarr_metadata import ZarrV3ArrayMetadataJSON + + +class AcmeAffineConfiguration(TypedDict, closed=True): + scale: float + offset: NotRequired[float] + dtype: NotRequired[ZarrV3MetadataFieldJSON] + + +class AcmeAffineObject(TypedDict, closed=True): + name: Literal["acme.affine"] + configuration: AcmeAffineConfiguration + must_understand: NotRequired[bool] + + +@dataclass(frozen=True) +class AcmeAffineCodec(CodecEntity[AcmeAffineObject]): + """`x * scale + offset`, stored as `dtype` if one is named.""" + + scale: float + offset: float | UNSET = UNSET + dtype: DataTypeEntity | Opaque | UNSET = UNSET + + identifier: ClassVar[str] = "acme.affine" + kind: ClassVar[CodecKind] = "array_array" + + def __post_init__(self) -> None: + found: list[ValidationProblem] = [] + if self.scale == 0: + found.extend(problem(("scale",), "expected a non-zero number, got 0", "invalid_value")) + if ( + isinstance(self.dtype, DataTypeEntity) + and self.dtype.storage_class() == "variable_length" + ): + found.extend( + problem( + ("dtype",), + f"expected a fixed-size data type, got {type(self.dtype).identifier!r}", + "invalid_value", + ) + ) + if len(found) != 0: + raise MetadataValidationError(found) + + def simplified(self) -> Self: + """An offset of 0 is the identity, and absent says the same.""" + return replace(self, offset=UNSET) if self.offset == 0 else self + + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + data_type = incoming.data_type if incoming is not None else None + if data_type is None or data_type.storage_class() != "variable_length": + return () + return problem( + (), + f"acme.affine cannot scale variable-length data_type {type(data_type).identifier!r}", + "invalid_value", + ) + + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + if self.dtype is UNSET: + return incoming + return incoming.with_data_type( + self.dtype if isinstance(self.dtype, DataTypeEntity) else None + ) + + +SCOPE = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeAffineCodec.identifier: AcmeAffineCodec}) +BYTES_LE = {"name": "bytes", "configuration": {"endian": "little"}} + + +def _affine(**configuration: object) -> dict[str, object]: + return {"name": "acme.affine", "configuration": configuration} + + +def _document(**overrides: object) -> dict[str, object]: + return { + "zarr_format": 3, + "node_type": "array", + "shape": [8], + "data_type": "float32", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [8]}}, + "chunk_key_encoding": "default", + "codecs": [_affine(scale=2), BYTES_LE], + **overrides, + } + + +@pytest.mark.parametrize( + "configuration", + [ + {"scale": 2}, + {"scale": 2.5, "offset": -1}, + {"scale": -0.5, "offset": 0.25, "dtype": "float64"}, + {"scale": 3, "offset": 0, "dtype": {"name": "float32"}}, + ], +) +def test_a_valid_document_has_no_problems(configuration: dict[str, object]) -> None: + document = _document(codecs=[_affine(**configuration), BYTES_LE]) + assert validate_array_metadata_v3(document, context=SCOPE) == () + assert isinstance(ArrayDocumentV3.from_json(document, context=SCOPE).codecs[0], AcmeAffineCodec) + + +def test_error_problems_are_located_at_their_members() -> None: + document = _document( + codecs=[ + {"name": "transpose", "configuration": {"order": [0]}}, + _affine(scale=0, offset=None), + BYTES_LE, + ] + ) + problems = validate_array_metadata_v3(document, context=SCOPE) + # A null offset fails the type check, so the value rules do not run + # on the hole: one problem, at the member that has it. + assert [(found.loc, found.kind) for found in problems] == [ + (("codecs", 1, "configuration", "offset"), "invalid_type") + ] + problems = validate_array_metadata_v3( + _document(codecs=[_affine(scale=0), BYTES_LE]), context=SCOPE + ) + assert [(found.loc, found.kind) for found in problems] == [ + (("codecs", 0, "configuration", "scale"), "invalid_value") + ] + + +def test_error_a_variable_length_type_is_refused_in_and_out() -> None: + document = _document(data_type="string", fill_value="", codecs=[_affine(scale=2), "vlen-utf8"]) + problems = validate_array_metadata_v3(document, context=SCOPE) + assert [(found.loc, found.kind) for found in problems] == [(("codecs", 0), "invalid_value")] + document = _document(codecs=[_affine(scale=2, dtype="string"), BYTES_LE]) + problems = validate_array_metadata_v3(document, context=SCOPE) + assert [(found.loc, found.kind) for found in problems] == [ + (("codecs", 0, "configuration", "dtype"), "invalid_value") + ] + + +def test_dtype_changes_what_the_next_codec_sees() -> None: + # uint8 needs no endianness and float32 does, so a bare `bytes` codec + # is fine before the transition and wrong after it. + without = _document(data_type="uint8", codecs=[_affine(scale=2), "bytes"]) + assert validate_array_metadata_v3(without, context=SCOPE) == () + retyped = _document(data_type="uint8", codecs=[_affine(scale=2, dtype="float32"), "bytes"]) + problems = validate_array_metadata_v3(retyped, context=SCOPE) + assert [(found.loc, found.kind) for found in problems] == [ + (("codecs", 1, "configuration", "endian"), "missing_key") + ] + + +def test_an_out_of_scope_dtype_is_kept_and_not_judged() -> None: + entry = _affine(scale=2, dtype="mycorp.decimal") + document = _document(codecs=[entry, "bytes"]) + assert validate_array_metadata_v3(document, context=SCOPE) == () + codec = ArrayDocumentV3.from_json(document, context=SCOPE).codecs[0] + assert isinstance(codec, AcmeAffineCodec) + assert isinstance(codec.dtype, Opaque) + assert codec.dtype.reason == "out_of_scope" + assert codec.to_json() == entry + + +def test_round_trip_and_canonical() -> None: + entry = _affine(scale=2, offset=1, dtype="float64") + codec = ArrayDocumentV3.from_json(_document(codecs=[entry, BYTES_LE]), context=SCOPE).codecs[0] + assert isinstance(codec, AcmeAffineCodec) + written: AcmeAffineObject = codec.to_json() + assert written == entry + assert AcmeAffineCodec(scale=2, offset=0).canonical() == AcmeAffineCodec(scale=2) + document = cast( + "ZarrV3ArrayMetadataJSON", _document(codecs=[_affine(scale=2, offset=0.0), BYTES_LE]) + ) + result = canonicalize_array_metadata_v3(document, context=SCOPE) + assert isinstance(result, Canonical) + assert cast("tuple[object, ...]", result.document["codecs"])[0] == _affine(scale=2) + + +def test_constructed_by_hand() -> None: + codec = AcmeAffineCodec(scale=2.5, offset=-1, dtype=Float32DataType()) + assert codec.to_json() == { + "name": "acme.affine", + "configuration": {"scale": 2.5, "offset": -1, "dtype": "float32"}, + } + with pytest.raises(MetadataValidationError) as caught: + AcmeAffineCodec(scale=0) + assert [(found.loc, found.kind) for found in caught.value.problems] == [ + (("scale",), "invalid_value") + ] diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py new file mode 100644 index 0000000000..6d85d9e86c --- /dev/null +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -0,0 +1,354 @@ +"""`acme.decimal`, a third-party data type, written against the door alone. + +A fixed-size decimal stored as a 16-byte integer. The configuration carries `precision` (1..38 significant digits) and +`scale` (0..precision digits after the point); a fill value is a JSON +string holding a decimal literal such as `"12.50"` whose digits fit both. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast + +import pytest +from typing_extensions import ReadOnly, TypedDict + +if TYPE_CHECKING: + from collections.abc import Iterable + + from zarr_metadata import ZarrV3ArrayMetadataJSON + +from zarr_metadata.v3.entity import ( + DataTypeEntity, + Loc, + MetadataValidationError, + StorageClass, + ValidationProblem, + problem, +) + +ACME_DECIMAL_DATA_TYPE_NAME: Final = "acme.decimal" +"""The `name` field value of the `acme.decimal` data type.""" + +AcmeDecimalDataTypeName = Literal["acme.decimal"] +"""Literal type of the `name` field of the `acme.decimal` data type.""" + +ACME_DECIMAL_MAX_PRECISION: Final = 38 +"""The most significant digits a 16-byte integer holds in every case.""" + +DECIMAL_LITERAL: Final = re.compile(r"-?(?P[0-9]+)(?:\.(?P[0-9]+))?") +"""A plain decimal literal: an optional sign, digits, an optional fraction.""" + + +class AcmeDecimalConfiguration(TypedDict, closed=True): + """Configuration for the `acme.decimal` data type.""" + + precision: ReadOnly[int] + scale: ReadOnly[int] + + +class AcmeDecimal(TypedDict, closed=True): + """`acme.decimal` data type metadata.""" + + name: AcmeDecimalDataTypeName + configuration: AcmeDecimalConfiguration + must_understand: NotRequired[bool] + + +AcmeDecimalFillValue = str +"""Permitted JSON shape of the `fill_value` field for `acme.decimal`: a decimal literal.""" + + +__all__ = [ + "ACME_DECIMAL_DATA_TYPE_NAME", + "ACME_DECIMAL_MAX_PRECISION", + "AcmeDecimal", + "AcmeDecimalConfiguration", + "AcmeDecimalDataType", + "AcmeDecimalDataTypeName", + "AcmeDecimalFillValue", +] + + +@dataclass(frozen=True) +class AcmeDecimalDataType(DataTypeEntity[AcmeDecimal]): + """The `acme.decimal` data type, coerced from its metadata.""" + + precision: int + scale: int + + identifier: ClassVar[str] = ACME_DECIMAL_DATA_TYPE_NAME + scalar_storage: ClassVar[StorageClass] = "multi_byte" + + def __post_init__(self) -> None: + found: list[ValidationProblem] = [] + if not 1 <= self.precision <= ACME_DECIMAL_MAX_PRECISION: + found.extend( + problem( + ("precision",), + f"expected an integer in [1, {ACME_DECIMAL_MAX_PRECISION}], " + f"got {self.precision}", + "invalid_value", + ) + ) + if self.scale < 0: + found.extend( + problem(("scale",), f"expected an integer >= 0, got {self.scale}", "invalid_value") + ) + elif self.scale > self.precision: + found.extend( + problem( + ("scale",), + f"expected an integer <= precision ({self.precision}), got {self.scale}", + "invalid_value", + ) + ) + if len(found) != 0: + raise MetadataValidationError(found) + + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + """A decimal literal whose digits fit `precision` and `scale`. + + Judged as written: `"12.50"` has two fractional digits whatever + its value, so it needs a scale of at least two. + """ + if not isinstance(value, str): + return problem( + loc, f"expected a decimal literal string, got {value!r}", "invalid_value" + ) + matched = DECIMAL_LITERAL.fullmatch(value) + if matched is None: + return problem( + loc, f"expected a decimal literal like '12.50', got {value!r}", "invalid_value" + ) + integer_digits = len(matched.group("integer").lstrip("0")) + fraction = matched.group("fraction") + fraction_digits = 0 if fraction is None else len(fraction) + allowed_integer_digits = self.precision - self.scale + found: list[ValidationProblem] = [] + if fraction_digits > self.scale: + found.extend( + problem( + loc, + f"{value!r} has {fraction_digits} fractional digits, but scale is {self.scale}", + "invalid_value", + ) + ) + if integer_digits > allowed_integer_digits: + found.extend( + problem( + loc, + f"{value!r} has {integer_digits} integer digits, but precision {self.precision} " + f"with scale {self.scale} allows {allowed_integer_digits}", + "invalid_value", + ) + ) + return tuple(found) + + +# --- the tests -------------------------------------------------------------- + +from zarr_metadata.rules import canonicalize_array_metadata_v3, validate_array_metadata_v3 +from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS, ArrayDocumentV3, Context, Opaque + +SCOPE: Context = CORE_AND_EXTENSIONS.extended_with( + data_type={AcmeDecimalDataType.identifier: AcmeDecimalDataType} +) + +LITTLE_ENDIAN_BYTES = {"name": "bytes", "configuration": {"endian": "little"}} + + +def _data_type(precision: int, scale: int) -> AcmeDecimal: + return { + "name": ACME_DECIMAL_DATA_TYPE_NAME, + "configuration": {"precision": precision, "scale": scale}, + } + + +def _document(**overrides: object) -> dict[str, object]: + return { + "zarr_format": 3, + "node_type": "array", + "shape": (8,), + "data_type": _data_type(4, 2), + "fill_value": "12.50", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (8,)}}, + "chunk_key_encoding": "default", + "codecs": (LITTLE_ENDIAN_BYTES,), + **overrides, + } + + +def _locs(problems: Iterable[ValidationProblem]) -> list[Loc]: + return [problem.loc for problem in problems] + + +# --- documents, judged in a scope that knows the type --------------------- + + +def test_a_valid_document_has_no_problems() -> None: + assert validate_array_metadata_v3(_document(), context=SCOPE) == () + + +def test_a_bare_bytes_codec_is_refused_for_a_sixteen_byte_type() -> None: + # A 16-byte integer has a byte order, so the `bytes` codec needs to be + # told which one: the same rule the core multi-byte types are held to. + problems = validate_array_metadata_v3(_document(codecs=("bytes",)), context=SCOPE) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("codecs", 0, "configuration", "endian"), "missing_key") + ] + + +def test_error_scale_above_precision_is_located_at_the_member() -> None: + document = _document(data_type=_data_type(4, 5)) + problems = validate_array_metadata_v3(document, context=SCOPE) + assert _locs(problems) == [("data_type", "configuration", "scale")] + + +def test_error_a_fill_value_that_does_not_fit_is_located_at_fill_value() -> None: + problems = validate_array_metadata_v3(_document(fill_value="1234.5"), context=SCOPE) + assert _locs(problems) == [("fill_value",)] + + +def test_the_type_passes_through_a_sharding_codec() -> None: + shard = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4,), + "codecs": (LITTLE_ENDIAN_BYTES,), + "index_codecs": (LITTLE_ENDIAN_BYTES, "crc32c"), + }, + } + document = _document(shape=(16,), codecs=(shard,)) + assert validate_array_metadata_v3(document, context=SCOPE) == () + array = ArrayDocumentV3.from_json(document, context=SCOPE) + assert isinstance(array.data_type, AcmeDecimalDataType) + + +def test_an_inner_bare_bytes_codec_is_refused_inside_a_shard_too() -> None: + shard = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4,), + "codecs": ("bytes",), + "index_codecs": (LITTLE_ENDIAN_BYTES, "crc32c"), + }, + } + problems = validate_array_metadata_v3(_document(shape=(16,), codecs=(shard,)), context=SCOPE) + assert _locs(problems) == [ + ("codecs", 0, "configuration", "codecs", 0, "configuration", "endian") + ] + + +def test_an_unregistered_scope_reads_it_as_opaque_and_judges_nothing() -> None: + document = _document(fill_value="this is not judged") + assert validate_array_metadata_v3(document) == () + array = ArrayDocumentV3.from_json(document) + assert isinstance(array.data_type, Opaque) + assert array.data_type.reason == "out_of_scope" + + +# --- round trip ------------------------------------------------------------- + + +def test_to_json_writes_back_what_was_read() -> None: + document = _document() + array = ArrayDocumentV3.from_json(document, context=SCOPE) + assert isinstance(array.data_type, DataTypeEntity) + assert array.data_type.to_json() == document["data_type"] + assert array.data_type.canonical() == array.data_type + + +def test_canonical_form_keeps_the_configuration() -> None: + document = cast("ZarrV3ArrayMetadataJSON", _document()) + result = canonicalize_array_metadata_v3(document, context=SCOPE) + assert result.valid is True + assert result.document["data_type"] == _data_type(4, 2) + + +# --- hand construction ------------------------------------------------------ + + +def test_hand_construction() -> None: + entity = AcmeDecimalDataType(precision=4, scale=2) + assert entity.storage_class() == "multi_byte" + assert entity.to_json() == _data_type(4, 2) + assert entity == AcmeDecimalDataType(4, 2) + assert hash(entity) == hash(AcmeDecimalDataType(4, 2)) + + +@pytest.mark.parametrize( + ("precision", "scale", "value"), + [ + (4, 2, "12.50"), + (4, 2, "-99.99"), + (4, 2, "0"), + (4, 2, "0.5"), + (4, 2, "0012.5"), + (4, 0, "1234"), + (4, 4, "0.1234"), + (1, 1, "0.0"), + (38, 10, "1234567890123456789012345678.0123456789"), + ], +) +def test_fill_values_that_fit_are_accepted(precision: int, scale: int, value: str) -> None: + entity = AcmeDecimalDataType(precision=precision, scale=scale) + assert entity.fill_value_problems(value, ("fill_value",)) == () + + +@pytest.mark.parametrize("precision", [0, 39]) +def test_error_precision_out_of_range(precision: int) -> None: + with pytest.raises(MetadataValidationError) as caught: + AcmeDecimalDataType(precision=precision, scale=0) + assert _locs(caught.value.problems) == [("precision",)] + + +def test_error_scale_negative() -> None: + with pytest.raises(MetadataValidationError) as caught: + AcmeDecimalDataType(precision=4, scale=-1) + assert _locs(caught.value.problems) == [("scale",)] + + +def test_error_scale_above_precision() -> None: + with pytest.raises(MetadataValidationError) as caught: + AcmeDecimalDataType(precision=4, scale=5) + assert [(problem.loc, problem.message) for problem in caught.value.problems] == [ + (("scale",), "expected an integer <= precision (4), got 5") + ] + + +def test_error_every_member_problem_is_reported_at_once() -> None: + with pytest.raises(MetadataValidationError) as caught: + AcmeDecimalDataType(precision=0, scale=-1) + assert _locs(caught.value.problems) == [("precision",), ("scale",)] + + +def test_error_fill_value_must_be_a_string() -> None: + entity = AcmeDecimalDataType(precision=4, scale=2) + assert _locs(entity.fill_value_problems(12.5, ("fill_value",))) == [("fill_value",)] + + +@pytest.mark.parametrize("value", ["", "1e2", " 12.5", "12.", ".5", "abc", "1,5", "NaN"]) +def test_error_fill_value_must_be_a_decimal_literal(value: str) -> None: + entity = AcmeDecimalDataType(precision=4, scale=2) + assert _locs(entity.fill_value_problems(value, ("fill_value",))) == [("fill_value",)] + + +def test_error_fill_value_with_too_many_fraction_digits() -> None: + entity = AcmeDecimalDataType(precision=4, scale=2) + problems = entity.fill_value_problems("1.234", ("fill_value",)) + assert [(problem.loc, problem.message) for problem in problems] == [ + (("fill_value",), "'1.234' has 3 fractional digits, but scale is 2") + ] + + +def test_error_fill_value_with_too_many_integer_digits() -> None: + entity = AcmeDecimalDataType(precision=4, scale=2) + problems = entity.fill_value_problems("1234.5", ("fill_value",)) + assert [(problem.loc, problem.message) for problem in problems] == [ + ( + ("fill_value",), + "'1234.5' has 4 integer digits, but precision 4 with scale 2 allows 2", + ) + ] diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 31ee49916f..e9c2310dc8 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -18,8 +18,6 @@ canonicalize_array_metadata_v3, validate_array_metadata_v3, ) -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -from zarr_metadata.v3._entity import json_type_of from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecObject from zarr_metadata.v3.entity import ( @@ -37,6 +35,7 @@ MetadataEntity, Opaque, StorageClass, + ZarrV3MetadataFieldJSON, problem, ) @@ -698,9 +697,8 @@ class Mismatched(CodecEntity[AcmeLvlObject]): # pyright: ignore[reportUnusedCla def test_a_third_party_entity_may_name_its_json_type_or_not() -> None: # Left defaulted, `to_json` is typed as any metadata field; named, as - # the entity's own type -- and either way the same dict comes back. - assert json_type_of(AcmeLz4Codec) is ZarrV3MetadataFieldJSON - + # the entity's own type, held to the members at class creation -- and + # either way the same dict comes back. @dataclass(frozen=True) class AcmeTypedBlockCodec(CodecEntity[AcmeBlockObject]): block: int @@ -708,7 +706,6 @@ class AcmeTypedBlockCodec(CodecEntity[AcmeBlockObject]): identifier: ClassVar[str] = "acme.block" kind: ClassVar[CodecKind] = "bytes_bytes" - assert json_type_of(AcmeTypedBlockCodec) is AcmeBlockObject assert AcmeTypedBlockCodec(block=8).to_json() == { "name": "acme.block", "configuration": {"block": 8}, From eea42d557fae900dc1e9bd3a111b06db2fb5e6f9 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 14:53:20 +0200 Subject: [PATCH 076/107] chore(zarr-metadata): every suppression names a pyright rule, or goes Pyright is the only checker that runs on this package -- the root mypy config covers the root package alone -- and it honours `# type: ignore` as a blanket suppression of every diagnostic on the line. So the 44 mypy-style comments here were unlabeled pyright suppressions: 17 hid nothing, and the rest hid real mismatches under codes that named a different checker's rule. All gone. The config now switches that spelling off and errors on a `# pyright: ignore` that suppresses nothing. What they were hiding, fixed at the source: `canonicalize_array_metadata_v3` takes `object` like its siblings, since it normalizes and judges structure first (seven casts and ignores go); tables of extension points are typed `ExtensionPointField`, so `coerce` and `resolve` get the literal their overloads want; the tests look into JSON through `tests/helpers.py` (`entry_at`, `configuration_of`) or by narrowing to the concrete entity, instead of indexing a `str | TypedDict`; the fill-value helper returns a `DataTypeEntity`; the location walker narrows each step to its container's key type. In `src`, the `Sequence[Unknown]` an `isinstance` leaves is cast to `Sequence[object]`. Six `# pyright: ignore[rule]` remain, each on a line that is wrong on purpose -- a guard test's deliberate mistake that pyright rightly refuses statically and the test proves is refused at run time -- or on a stub that cannot express the call (hypothesis's `register_type_strategy` with a `TypeAliasType`). Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/pyproject.toml | 7 +++ .../src/zarr_metadata/rules/_documents.py | 10 +-- .../zarr_metadata/v3/data_type/_families.py | 5 +- packages/zarr-metadata/tests/helpers.py | 28 +++++++++ .../zarr-metadata/tests/rules/strategies.py | 2 +- .../tests/rules/test_canonical.py | 23 +++---- .../tests/rules/test_chunk_grid.py | 26 +++++--- .../tests/v3/test_acme_affine.py | 11 +--- .../tests/v3/test_acme_decimal.py | 6 +- .../zarr-metadata/tests/v3/test_entities.py | 63 +++++++++++-------- .../tests/v3/test_extension_api.py | 47 +++++++------- .../tests/v3/test_fill_values.py | 13 ++-- .../zarr-metadata/tests/v3/test_resolve.py | 18 ++++-- 13 files changed, 157 insertions(+), 102 deletions(-) create mode 100644 packages/zarr-metadata/tests/helpers.py diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 8bdf578cf4..958d5fec77 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -129,6 +129,13 @@ checks = [ # keeps a pyright release from turning CI red on its own schedule. [tool.pyright] include = ["src", "tests"] +# Pyright is the only checker that runs on this package (the root mypy +# config covers the root package alone), and it honours `# type: ignore` +# as a blanket suppression of every diagnostic on the line. So that +# spelling is switched off: a suppression is `# pyright: ignore[rule]`, +# names the rule, and is an error once it stops suppressing anything. +enableTypeIgnoreComments = false +reportUnnecessaryTypeIgnoreComment = "error" # `tests` is a package imported as `tests.*`, which pytest resolves from the # rootdir; pyright needs the same root on its search path. extraPaths = ["."] diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index 5695209a51..61a60709e0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -226,7 +226,7 @@ def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: def canonicalize_array_metadata_v3( - document: ZarrV3ArrayMetadataJSON, *, context: Context = CORE_AND_EXTENSIONS + document: object, *, context: Context = CORE_AND_EXTENSIONS ) -> Canonical[ZarrV3ArrayMetadataJSON] | Invalid: """`document` in canonical form, or every reason it is not valid. @@ -237,10 +237,10 @@ def canonicalize_array_metadata_v3( `tests/rules/test_canonical.py` asserts both: canonicalizing twice changes nothing further, and canonicalizing never changes a verdict. - Expects a document the model layer has already accepted. Passing one - it has not is not an error -- the semantic problems are reported the - same way -- but the structural problems come back too, and the result - is `Invalid` rather than a canonical document. The document is read + Takes any value, like `validate_array_metadata_v3`: a document the + model layer has not accepted is not an error -- the structural + problems come back with the semantic ones, and the result is + `Invalid` rather than a canonical document. The document is read once: the entities that judge it are the entities that are rewritten. """ normalized = arrays_to_tuples(document) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 4645732a62..c87e93c149 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -15,7 +15,7 @@ from collections.abc import Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Final, Literal +from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( @@ -43,7 +43,8 @@ def as_sequence(value: object) -> tuple[object, ...] | None: """ if isinstance(value, str) or not isinstance(value, Sequence): return None - return tuple(value) # type: ignore[arg-type] + # `isinstance` narrows to `Sequence[Unknown]`; its elements are objects. + return tuple(cast("Sequence[object]", value)) def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/tests/helpers.py b/packages/zarr-metadata/tests/helpers.py new file mode 100644 index 0000000000..ccc6294790 --- /dev/null +++ b/packages/zarr-metadata/tests/helpers.py @@ -0,0 +1,28 @@ +"""Ways for a test to look into JSON it has just been handed, without lying about its type.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import cast + + +def entry_at(document: object, *steps: str | int) -> object: + """The value `steps` lead to in `document`: a key into an object, an index into an array.""" + node = document + for step in steps: + if isinstance(step, str): + assert isinstance(node, Mapping), (steps, node) + node = cast("Mapping[str, object]", node)[step] + else: + assert isinstance(node, Sequence), (steps, node) + assert not isinstance(node, str), (steps, node) + node = cast("Sequence[object]", node)[step] + return node + + +def configuration_of(field: object) -> Mapping[str, object]: + """The configuration of a metadata field spelled as an object; empty if it has none.""" + assert isinstance(field, Mapping), field + configuration = cast("Mapping[str, object]", field).get("configuration", {}) + assert isinstance(configuration, Mapping), configuration + return cast("Mapping[str, object]", configuration) diff --git a/packages/zarr-metadata/tests/rules/strategies.py b/packages/zarr-metadata/tests/rules/strategies.py index 1884af8e31..7bd9750b5a 100644 --- a/packages/zarr-metadata/tests/rules/strategies.py +++ b/packages/zarr-metadata/tests/rules/strategies.py @@ -60,7 +60,7 @@ # `JSONValue` is a `TypeAliasType`, which `register_type_strategy` does not # accept in its signature but does resolve at runtime — it is exactly the # forward reference `from_type` fails on. -st.register_type_strategy(JSONValue, JSON_VALUES) # type: ignore[arg-type] +st.register_type_strategy(JSONValue, JSON_VALUES) # pyright: ignore[reportArgumentType] # The codec TypedDicts, by pipeline kind. Hand-written because there is no # name-to-type table to derive it from; `test_chain_properties.py` asserts it diff --git a/packages/zarr-metadata/tests/rules/test_canonical.py b/packages/zarr-metadata/tests/rules/test_canonical.py index d5da358af1..78b8efaf2a 100644 --- a/packages/zarr-metadata/tests/rules/test_canonical.py +++ b/packages/zarr-metadata/tests/rules/test_canonical.py @@ -13,6 +13,7 @@ import pytest from hypothesis import HealthCheck, given, settings +from tests.helpers import configuration_of, entry_at from tests.rules.strategies import valid_documents from zarr_metadata.rules import ( Canonical, @@ -39,7 +40,7 @@ def _canonical(**overrides: object) -> Mapping[str, object]: - result = canonicalize_array_metadata_v3({**BASE, **overrides}) # type: ignore[arg-type] + result = canonicalize_array_metadata_v3({**BASE, **overrides}) assert isinstance(result, Canonical), result return result.document @@ -90,7 +91,7 @@ def test_error_an_extension_point_may_not_be_declared_ignorable() -> None: # it, and no extension point is skippable -- so there is nothing for # canonicalization to keep. result = canonicalize_array_metadata_v3( - {**BASE, "codecs": ({"name": "bytes", "must_understand": False},)} # type: ignore[arg-type] + {**BASE, "codecs": ({"name": "bytes", "must_understand": False},)} ) assert isinstance(result, Invalid) assert [problem.loc for problem in result.problems] == [("codecs", 0, "must_understand")] @@ -103,15 +104,15 @@ def test_blosc_drops_a_typesize_that_shuffle_renders_ignored() -> None: "bytes", {"name": "blosc", "configuration": {**configuration, "shuffle": "noshuffle"}}, ) - )["codecs"][1] # type: ignore[index] - assert "typesize" not in dropped["configuration"] # type: ignore[index] + ) + assert "typesize" not in configuration_of(entry_at(dropped, "codecs", 1)) kept = _canonical( codecs=( "bytes", {"name": "blosc", "configuration": {**configuration, "shuffle": "shuffle"}}, ) - )["codecs"][1] # type: ignore[index] - assert kept["configuration"]["typesize"] == 4 # type: ignore[index] + ) + assert configuration_of(entry_at(kept, "codecs", 1))["typesize"] == 4 def test_dimension_names_of_nothing_but_nulls_are_dropped() -> None: @@ -128,13 +129,13 @@ def test_a_rectilinear_step_is_not_expanded() -> None: "configuration": {"kind": "inline", "chunk_shapes": (spec,)}, } shape = (64,) if spec == 32 else (32,) - result = canonicalize_array_metadata_v3({**BASE, "shape": shape, "chunk_grid": grid}) # type: ignore[arg-type] + result = canonicalize_array_metadata_v3({**BASE, "shape": shape, "chunk_grid": grid}) assert isinstance(result, Canonical), result - assert result.document["chunk_grid"]["configuration"]["chunk_shapes"] == (spec,) # type: ignore[index] + assert entry_at(result.document, "chunk_grid", "configuration", "chunk_shapes") == (spec,) def test_error_a_semantically_invalid_document_reports_instead() -> None: - result = canonicalize_array_metadata_v3({**BASE, "fill_value": 999}) # type: ignore[arg-type] + result = canonicalize_array_metadata_v3({**BASE, "fill_value": 999}) assert isinstance(result, Invalid) # The field is the location, not part of the message: the data type # says what it accepts, and the document says where it was asked. @@ -149,7 +150,7 @@ def test_error_invalid_cannot_be_empty() -> None: @given(valid_documents()) @_SLOW def test_canonicalizing_twice_changes_nothing_further(doc: Mapping[str, object]) -> None: - once = canonicalize_array_metadata_v3(doc) # type: ignore[arg-type] + once = canonicalize_array_metadata_v3(doc) assert isinstance(once, Canonical), once twice = canonicalize_array_metadata_v3(once.document) assert isinstance(twice, Canonical), twice @@ -161,6 +162,6 @@ def test_canonicalizing_twice_changes_nothing_further(doc: Mapping[str, object]) def test_canonicalizing_never_changes_the_verdict(doc: Mapping[str, object]) -> None: # A simplification that changed meaning would show up here as a # document that validated before and does not after. - result = canonicalize_array_metadata_v3(doc) # type: ignore[arg-type] + result = canonicalize_array_metadata_v3(doc) assert isinstance(result, Canonical), result assert validate_array_metadata_v3(result.document) == () diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index 2b2f19daa7..435e665120 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -2,17 +2,15 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from collections.abc import Mapping import pytest +from tests.helpers import configuration_of, entry_at from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._parts import ChunkGrid, shard_index_grid from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS -if TYPE_CHECKING: - from collections.abc import Mapping - BASE: Mapping[str, object] = { "zarr_format": 3, "node_type": "array", @@ -94,14 +92,20 @@ def _grid_of(grid: object, shape: object) -> ChunkGrid: A grid entity builds its own; one out of scope pins only the rank the array shape gives it. """ - name = grid if isinstance(grid, str) else (grid or {}).get("name") # type: ignore[union-attr] + name = ( + grid + if isinstance(grid, str) + else entry_at(grid, "name") + if isinstance(grid, Mapping) + else None + ) entity_type = CORE_AND_EXTENSIONS.resolve("chunk_grid", name) if isinstance(name, str) else None if entity_type is None: return ChunkGrid.unreadable(shape) entity, _ = entity_type.coerce(grid, CORE_AND_EXTENSIONS) if entity is None: return ChunkGrid.unreadable(shape) - return entity.grid(shape) # type: ignore[attr-defined] + return entity.grid(shape) @pytest.mark.parametrize(("grid", "shape", "rank", "extents"), GRIDS.values(), ids=list(GRIDS)) @@ -209,10 +213,12 @@ def test_error_a_bad_inner_extent_costs_the_shard() -> None: # exist and nothing inside it is interpreted. The JSON is still there # on the `Opaque` that replaces it; what is gone is the reading, and # the one report that matters is the one you must fix first. - inner = _shard((0, 2)) - inner["configuration"] = { # type: ignore[index] - **inner["configuration"], # type: ignore[dict-item] - "codecs": ({"name": "transpose", "configuration": {"order": (0, 1, 2)}}, "bytes"), + inner = { + **_shard((0, 2)), + "configuration": { + **configuration_of(_shard((0, 2))), + "codecs": ({"name": "transpose", "configuration": {"order": (0, 1, 2)}}, "bytes"), + }, } problems = validate_array_metadata_v3( {**BASE, "data_type": "uint16", "chunk_grid": REGULAR, "codecs": (inner,)} diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 22eb71cd17..b59c61ebb9 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, Self, cast +from typing import ClassVar, Literal, NotRequired, Self import pytest from typing_extensions import TypedDict @@ -35,9 +35,6 @@ problem, ) -if TYPE_CHECKING: - from zarr_metadata import ZarrV3ArrayMetadataJSON - class AcmeAffineConfiguration(TypedDict, closed=True): scale: float @@ -202,12 +199,10 @@ def test_round_trip_and_canonical() -> None: written: AcmeAffineObject = codec.to_json() assert written == entry assert AcmeAffineCodec(scale=2, offset=0).canonical() == AcmeAffineCodec(scale=2) - document = cast( - "ZarrV3ArrayMetadataJSON", _document(codecs=[_affine(scale=2, offset=0.0), BYTES_LE]) - ) + document = _document(codecs=[_affine(scale=2, offset=0.0), BYTES_LE]) result = canonicalize_array_metadata_v3(document, context=SCOPE) assert isinstance(result, Canonical) - assert cast("tuple[object, ...]", result.document["codecs"])[0] == _affine(scale=2) + assert result.document["codecs"][0] == _affine(scale=2) def test_constructed_by_hand() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 6d85d9e86c..f8215e2623 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -9,7 +9,7 @@ import re from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired import pytest from typing_extensions import ReadOnly, TypedDict @@ -17,7 +17,6 @@ if TYPE_CHECKING: from collections.abc import Iterable - from zarr_metadata import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.entity import ( DataTypeEntity, @@ -261,8 +260,7 @@ def test_to_json_writes_back_what_was_read() -> None: def test_canonical_form_keeps_the_configuration() -> None: - document = cast("ZarrV3ArrayMetadataJSON", _document()) - result = canonicalize_array_metadata_v3(document, context=SCOPE) + result = canonicalize_array_metadata_v3(_document(), context=SCOPE) assert result.valid is True assert result.document["data_type"] == _data_type(4, 2) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 194ef41d14..b929746e62 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -21,6 +21,7 @@ import pytest from hypothesis import given, settings +from tests.helpers import configuration_of from tests.rules.strategies import valid_documents from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 @@ -74,7 +75,7 @@ from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType -from zarr_metadata.v3.entity import ArrayDocumentV3, MetadataEntity +from zarr_metadata.v3.entity import ArrayDocumentV3, ExtensionPointField, MetadataEntity # Every registered entity, keyed by `:` -- an identifier # is unique only within its extension point, and `bytes` is both a codec @@ -362,7 +363,7 @@ def test_a_nested_metadata_field_is_judged_like_a_top_level_one( }, ), } - problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document) assert [problem.loc for problem in problems] == [ ("codecs", 0, "configuration", "codecs", 0, *inner_loc) ] @@ -393,18 +394,24 @@ def test_every_problem_location_indexes_into_the_document() -> None: }, ), } - problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document) assert len(problems) != 0 for problem in problems: node: object = document for step in problem.loc: - if not isinstance(node, (dict, tuple)) or (isinstance(node, dict) and step not in node): - # A `missing_key` problem names where the key belongs, so - # it is allowed to run past the end of what is there. Any - # other kind must address a node that exists. - assert problem.kind == "missing_key", (problem.loc, step) - break - node = node[step] # type: ignore[index] + # Two branches rather than one `or`: each narrows `step` to + # the key type its container takes. + if isinstance(node, dict) and isinstance(step, str) and step in node: + node = node[step] + continue + if isinstance(node, tuple) and isinstance(step, int) and step < len(node): + node = node[step] + continue + # A `missing_key` problem names where the key belongs, so it + # is allowed to run past the end of what is there. Any other + # kind must address a node that exists. + assert problem.kind == "missing_key", (problem.loc, step) + break def test_an_unreadable_member_costs_the_entity() -> None: @@ -433,7 +440,7 @@ def test_an_unreadable_member_costs_the_entity() -> None: }, ), } - problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document) assert {problem.loc for problem in problems} == { ("codecs", 1, "configuration", "clevel"), } @@ -465,7 +472,7 @@ def test_an_optional_member_of_the_wrong_type_is_not_judged_as_absent() -> None: }, ), } - problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document) assert [(problem.loc, problem.kind) for problem in problems] == [ (("codecs", 1, "configuration", "typesize"), "invalid_type") ] @@ -504,12 +511,12 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: }, ), } - problems = validate_array_metadata_v3(document) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document) assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "shuffle")] # Entities whose written form and canonical form differ, or could. -FAITHFUL: dict[str, tuple[str, object]] = { +FAITHFUL: dict[str, tuple[ExtensionPointField, object]] = { "rectilinear-expanded": ( "chunk_grid", { @@ -555,11 +562,11 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: @pytest.mark.parametrize(("field", "written"), FAITHFUL.values(), ids=list(FAITHFUL)) -def test_to_json_writes_back_what_was_read(field: str, written: object) -> None: +def test_to_json_writes_back_what_was_read(field: ExtensionPointField, written: object) -> None: # Serialization is not canonicalization. A reader that reads a # document and writes it back must not change bytes it was not asked # to change -- `canonical()` is where you ask. - entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) # type: ignore[arg-type] + entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) assert problems == () assert isinstance(entity, MetadataEntity) assert entity.to_json() == written @@ -591,8 +598,8 @@ def test_canonical_is_what_simplifies() -> None: }, }, ) - assert isinstance(blosc, MetadataEntity) - assert "typesize" not in blosc.canonical().to_json()["configuration"] # type: ignore[index] + assert isinstance(blosc, BloscCodec) + assert "typesize" not in blosc.canonical().to_json()["configuration"] def test_canonical_reaches_a_contained_entity() -> None: @@ -619,9 +626,9 @@ def test_canonical_reaches_a_contained_entity() -> None: }, }, ) - assert isinstance(shard, MetadataEntity) - inner = shard.canonical().to_json()["configuration"]["codecs"][1] # type: ignore[index] - assert "typesize" not in inner["configuration"] # type: ignore[index] + assert isinstance(shard, ShardingIndexedCodec) + inner = shard.canonical().to_json()["configuration"]["codecs"][1] + assert "typesize" not in configuration_of(inner) def test_error_an_explicit_null_scalar_is_refused() -> None: @@ -636,7 +643,7 @@ def test_error_an_explicit_null_scalar_is_refused() -> None: # (an entity whose configuration holds a mutable JSON value) -MUTABLE_MEMBERS: dict[str, tuple[str, object]] = { +MUTABLE_MEMBERS: dict[str, tuple[ExtensionPointField, object]] = { "scale-offset-object": ( "codecs", {"name": "scale_offset", "configuration": {"offset": {"a": 1}}}, @@ -656,16 +663,18 @@ def test_error_an_explicit_null_scalar_is_refused() -> None: @pytest.mark.parametrize(("field", "written"), MUTABLE_MEMBERS.values(), ids=list(MUTABLE_MEMBERS)) -def test_to_json_shares_no_mutable_state_with_the_entity(field: str, written: object) -> None: +def test_to_json_shares_no_mutable_state_with_the_entity( + field: ExtensionPointField, written: object +) -> None: # The model layer has this test; the entity layer did not, and handed # out its own dict -- so a caller mutating the document it was given # mutated a frozen entity. - entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) # type: ignore[arg-type] + entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) assert problems == () assert isinstance(entity, MetadataEntity) baseline = copy.deepcopy(entity.to_json()) handed_out = entity.to_json() - configuration = handed_out["configuration"] # type: ignore[index] + configuration = configuration_of(handed_out) assert isinstance(configuration, dict) for key in list(configuration): value = configuration[key] @@ -694,8 +703,8 @@ def test_a_member_the_entity_does_not_model_is_not_written_back() -> None: } codec, problems = CORE_AND_EXTENSIONS.coerce("codecs", entry) assert [(p.loc, p.kind) for p in problems] == [(("configuration", "typo_key"), "unknown_key")] - assert isinstance(codec, MetadataEntity) - assert "typo_key" not in codec.to_json()["configuration"] # type: ignore[index,operator] + assert isinstance(codec, BloscCodec) + assert "typo_key" not in codec.to_json()["configuration"] def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index e9c2310dc8..f04e2a7880 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -103,7 +103,7 @@ def test_an_unregistered_name_is_not_judged() -> None: {"name": "acme.lz4", "configuration": {"acceleration": 999999}}, ) ) - assert validate_array_metadata_v3(document) == () # type: ignore[arg-type] + assert validate_array_metadata_v3(document) == () def test_a_registered_entity_is_judged() -> None: @@ -113,7 +113,7 @@ def test_a_registered_entity_is_judged() -> None: {"name": "acme.lz4", "configuration": {"acceleration": 999999}}, ) ) - problems = validate_array_metadata_v3(document, context=SCOPE) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document, context=SCOPE) assert [problem.loc for problem in problems] == [("codecs", 1, "configuration", "acceleration")] @@ -123,14 +123,14 @@ def test_a_registered_entity_joins_the_pipeline_rules() -> None: document = _document( codecs=("acme.lz4", {"name": "bytes", "configuration": {"endian": "little"}}) ) - problems = validate_array_metadata_v3(document, context=SCOPE) # type: ignore[arg-type] + problems = validate_array_metadata_v3(document, context=SCOPE) assert [problem.loc for problem in problems] == [("codecs", 1)] def test_a_registered_data_type_drives_the_codecs_around_it() -> None: # Single-byte, so the `bytes` codec needs no endianness for it. document = _document(data_type="acme.float8", fill_value=0, codecs=("bytes",)) - assert validate_array_metadata_v3(document, context=SCOPE) == () # type: ignore[arg-type] + assert validate_array_metadata_v3(document, context=SCOPE) == () def test_a_registered_entity_canonicalizes_itself() -> None: @@ -140,9 +140,9 @@ def test_a_registered_entity_canonicalizes_itself() -> None: {"name": "acme.lz4", "configuration": {}}, ) ) - result = canonicalize_array_metadata_v3(document, context=SCOPE) # type: ignore[arg-type] + result = canonicalize_array_metadata_v3(document, context=SCOPE) assert result.valid is True - assert result.document["codecs"][1] == "acme.lz4" # type: ignore[index] + assert result.document["codecs"][1] == "acme.lz4" def test_error_an_entity_must_say_what_it_is() -> None: @@ -150,7 +150,7 @@ def test_error_an_entity_must_say_what_it_is() -> None: # Never bound: the guard raises while the class is being created, # which is the whole point -- so pyright cannot see it used. @dataclass(frozen=True) - class Nameless(CodecEntity): # pyright: ignore[reportUnusedClass] + class Nameless(CodecEntity): kind: ClassVar[CodecKind] = "bytes_bytes" @@ -168,7 +168,8 @@ def test_error_an_entity_cannot_be_registered_at_the_wrong_point() -> None: # codec under `data_type` would resolve and then be asked for a # storage class it has no answer to. with pytest.raises(TypeError, match="registered at 'data_type', which takes DataTypeEntity"): - CORE.extended_with(data_type={AcmeLz4Codec.identifier: AcmeLz4Codec}) # type: ignore[dict-item] + # Deliberately wrong, and pyright says so; the runtime refusal is what is under test. + CORE.extended_with(data_type={AcmeLz4Codec.identifier: AcmeLz4Codec}) # pyright: ignore[reportArgumentType] def test_the_entity_layer_answers_what_a_reader_needs() -> None: @@ -195,10 +196,10 @@ def test_error_an_optional_member_defaults_to_unset() -> None: with pytest.raises(TypeError, match="a default other than UNSET"): @dataclass(frozen=True) - class Inventive(CodecEntity): # pyright: ignore[reportUnusedClass] + class Inventive(CodecEntity): # Optional by its type, so the annotation and the default agree # on that much; it is the default's value that is wrong. - level: int | UNSET = 3 # pyright: ignore[reportAssignmentType] + level: int | UNSET = 3 identifier: ClassVar[str] = "acme.inventive" kind: ClassVar[CodecKind] = "bytes_bytes" @@ -272,7 +273,7 @@ def test_error_a_field_may_not_shadow_a_class_variable() -> None: with pytest.raises(TypeError, match="shadowing a class variable"): @dataclass(frozen=True) - class Negotiable(CodecEntity): # pyright: ignore[reportUnusedClass] + class Negotiable(CodecEntity): must_understand: bool = True # pyright: ignore[reportIncompatibleVariableOverride] identifier: ClassVar[str] = "acme.negotiable" @@ -286,7 +287,7 @@ def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: with pytest.raises(TypeError, match="does not declare bounds"): @dataclass(frozen=True) - class Int24DataType(IntegerDataType): # pyright: ignore[reportUnusedClass] + class Int24DataType(IntegerDataType): identifier: ClassVar[str] = "acme.int24" @@ -339,7 +340,7 @@ def test_error_a_member_needs_a_check_from_somewhere() -> None: with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): @dataclass(frozen=True) - class Structured(CodecEntity): # pyright: ignore[reportUnusedClass] + class Structured(CodecEntity): inner: object identifier: ClassVar[str] = "acme.structured" @@ -440,7 +441,7 @@ def test_error_an_entity_may_not_override_canonical() -> None: with pytest.raises(TypeError, match="put the entity's own rewrite in `simplified`"): @dataclass(frozen=True) - class Rewriter(CodecEntity): # pyright: ignore[reportUnusedClass] + class Rewriter(CodecEntity): identifier: ClassVar[str] = "acme.rewriter" kind: ClassVar[CodecKind] = "bytes_bytes" @@ -476,7 +477,7 @@ def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: with pytest.raises(TypeError, match="has no `extension_point`"): @dataclass(frozen=True) - class Vague(CodecEntity): # pyright: ignore[reportUnusedClass] + class Vague(CodecEntity): inner: MetadataEntity | Opaque identifier: ClassVar[str] = "acme.vague" @@ -617,7 +618,7 @@ def test_error_a_nested_field_admits_opaque() -> None: with pytest.raises(TypeError, match="inner holds an entity but does not admit Opaque"): @dataclass(frozen=True) - class Closed(CodecEntity): # pyright: ignore[reportUnusedClass] + class Closed(CodecEntity): inner: CodecEntity identifier: ClassVar[str] = "acme.closed" @@ -629,7 +630,7 @@ def test_error_an_array_array_codec_defines_transition() -> None: with pytest.raises(TypeError, match="array_array codec and does not define transition"): @dataclass(frozen=True) - class Silent(CodecEntity): # pyright: ignore[reportUnusedClass] + class Silent(CodecEntity): identifier: ClassVar[str] = "acme.silent" kind: ClassVar[CodecKind] = "array_array" @@ -642,16 +643,16 @@ def test_error_a_literal_class_variable_holds_a_listed_value() -> None: ): @dataclass(frozen=True) - class Wide(DataTypeEntity): # pyright: ignore[reportUnusedClass] + class Wide(DataTypeEntity): identifier: ClassVar[str] = "acme.wide" - scalar_storage: ClassVar[StorageClass] = "sixteen_bytes" # type: ignore[assignment] # pyright: ignore[reportAssignmentType] + scalar_storage: ClassVar[StorageClass] = "sixteen_bytes" # pyright: ignore[reportAssignmentType] def test_error_a_list_of_problem_tuples_is_refused() -> None: # `problem()` returns a one-element tuple; a list of those would pass # the constructor and fail inside `coerce`, far from the mistake. with pytest.raises(TypeError, match="collect with `extend`, not `append`"): - MetadataValidationError([problem(("a",), "bad a")]) # type: ignore[list-item] # pyright: ignore[reportArgumentType] + MetadataValidationError([problem(("a",), "bad a")]) # pyright: ignore[reportArgumentType] def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: @@ -662,7 +663,7 @@ def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: ): @dataclass(frozen=True) - class Misnamed(CodecEntity[Literal["acme.misnamed"]]): # pyright: ignore[reportUnusedClass] + class Misnamed(CodecEntity[Literal["acme.misnamed"]]): level: int identifier: ClassVar[str] = "acme.misnamed" @@ -675,7 +676,7 @@ def test_error_the_named_json_type_must_name_what_the_entity_accepts() -> None: with pytest.raises(TypeError, match="names 'gzip', which the entity does not accept"): @dataclass(frozen=True) - class Impostor(CodecEntity[GzipCodecObject]): # pyright: ignore[reportUnusedClass] + class Impostor(CodecEntity[GzipCodecObject]): level: int identifier: ClassVar[str] = "acme.impostor" @@ -688,7 +689,7 @@ def test_error_the_named_json_type_must_have_the_members_as_keys() -> None: ): @dataclass(frozen=True) - class Mismatched(CodecEntity[AcmeLvlObject]): # pyright: ignore[reportUnusedClass] + class Mismatched(CodecEntity[AcmeLvlObject]): level: int identifier: ClassVar[str] = "acme.lvl" diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py index 37a18d1575..eb7e762d55 100644 --- a/packages/zarr-metadata/tests/v3/test_fill_values.py +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -9,6 +9,7 @@ import pytest +from tests.helpers import entry_at from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS from zarr_metadata.v3.entity import DataTypeEntity @@ -71,23 +72,25 @@ } -def _data_type(metadata: object) -> object: - name = metadata if isinstance(metadata, str) else metadata["name"] # type: ignore[index] - entity_type = CORE_AND_EXTENSIONS.resolve("data_type", name) # type: ignore[arg-type] +def _data_type(metadata: object) -> DataTypeEntity: + name = metadata if isinstance(metadata, str) else entry_at(metadata, "name") + assert isinstance(name, str), metadata + entity_type = CORE_AND_EXTENSIONS.resolve("data_type", name) assert entity_type is not None, metadata entity, problems = entity_type.coerce(metadata, CORE_AND_EXTENSIONS) assert problems == (), problems + assert entity is not None, metadata return entity @pytest.mark.parametrize(("metadata", "fill"), ACCEPTED.values(), ids=list(ACCEPTED)) def test_accepts(metadata: object, fill: object) -> None: - assert _data_type(metadata).fill_value_problems(fill) == () # type: ignore[attr-defined] + assert _data_type(metadata).fill_value_problems(fill) == () @pytest.mark.parametrize(("metadata", "fill", "reason"), REJECTED.values(), ids=list(REJECTED)) def test_error_rejects(metadata: object, fill: object, reason: str) -> None: - problems = _data_type(metadata).fill_value_problems(fill) # type: ignore[attr-defined] + problems = _data_type(metadata).fill_value_problems(fill) assert problems, f"expected {fill!r} to be rejected" assert any(reason in problem.message for problem in problems), problems diff --git a/packages/zarr-metadata/tests/v3/test_resolve.py b/packages/zarr-metadata/tests/v3/test_resolve.py index 3d9755235e..500e21229f 100644 --- a/packages/zarr-metadata/tests/v3/test_resolve.py +++ b/packages/zarr-metadata/tests/v3/test_resolve.py @@ -23,11 +23,12 @@ CODECS, CORE_AND_EXTENSIONS, DATA_TYPE, + ExtensionPointField, MetadataEntity, ) # (field, name, the entity that answers for it — None when nothing does) -RESOLUTIONS: dict[str, tuple[str, str, type[MetadataEntity] | None]] = { +RESOLUTIONS: dict[str, tuple[ExtensionPointField, str, type[MetadataEntity] | None]] = { "plain-dtype": (DATA_TYPE, "uint8", Uint8DataType), "dotted-dtype": (DATA_TYPE, "numpy.datetime64", NumpyDatetime64DataType), "raw-8": (DATA_TYPE, "r8", RawBytesDataType), @@ -48,9 +49,9 @@ @pytest.mark.parametrize(("field", "name", "expected"), RESOLUTIONS.values(), ids=list(RESOLUTIONS)) def test_a_name_resolves_to_the_entity_that_answers_for_it( - field: str, name: str, expected: type[MetadataEntity] | None + field: ExtensionPointField, name: str, expected: type[MetadataEntity] | None ) -> None: - assert CORE_AND_EXTENSIONS.resolve(field, name) is expected # type: ignore[arg-type] + assert CORE_AND_EXTENSIONS.resolve(field, name) is expected @given(width=st.integers(min_value=0, max_value=2**32)) @@ -62,11 +63,16 @@ def test_every_numeric_r_spelling_resolves_to_the_family(width: int) -> None: assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, f"r{width}") is RawBytesDataType -@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from([CODECS, CHUNK_GRID])) -def test_r_shaped_names_resolve_to_nothing_outside_data_types(width: int, field: str) -> None: +OTHER_POINTS: tuple[ExtensionPointField, ...] = (CODECS, CHUNK_GRID) + + +@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from(OTHER_POINTS)) +def test_r_shaped_names_resolve_to_nothing_outside_data_types( + width: int, field: ExtensionPointField +) -> None: # The family belongs to `data_type`; a codec that happens to be named # `r8` must not reach it. - assert CORE_AND_EXTENSIONS.resolve(field, f"r{width}") is None # type: ignore[arg-type] + assert CORE_AND_EXTENSIONS.resolve(field, f"r{width}") is None # The scan `resolve` falls back to asks every entity, so a name no entity From 6a473ab0ad49bd8903456cc82e480f5ec4837177 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 15:02:41 +0200 Subject: [PATCH 077/107] refactor(zarr-metadata): a class is not written to after its definition Review comments on `_entity.py`. The tables the layer reads off an entity's fields -- members and their checks, nested fields, envelope- named fields, whether a configuration is required -- were assigned onto the class in `__init_subclass__`; they are now `compiled(cls)`, a frozen record computed once per class and kept in a weak side table beside it. The hook only verifies. With no class variables to overwrite, the guard against declaring them goes, and the slots rebuild needs no special case beyond being recognised. The compiler no longer learns about entities through a module attribute `_entity` assigned at import: `MetadataFieldValue`, a behaviourless base in `_compile`, is what `MetadataEntity` and `Opaque` derive from, and a field typed as one is a nested metadata field. `dataclasses.MISSING` is the sentinel for "declared no default" instead of a bare `object()`. Prose that described the code relative to what it replaced is rewritten to say what the code is: `Opaque`, `_resolve`, the families module, and an orphaned comment about slots. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_compile.py | 61 ++-- .../src/zarr_metadata/v3/_entity.py | 284 +++++++----------- .../zarr_metadata/v3/data_type/_families.py | 6 +- .../tests/v3/test_extension_api.py | 19 +- 4 files changed, 162 insertions(+), 208 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py index fb7160dbfd..b33e344829 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py @@ -10,8 +10,9 @@ -- a bound, a rule about a member, members read together -- is the entity's own `__post_init__`, in plain code. -Nothing here knows what an entity is. A nested metadata field is the one -shape recognised through `nested_field`, which `_entity` sets. +Nothing here knows what an entity is. A nested metadata field is a +field typed as a class deriving from `MetadataFieldValue`, which is the +one thing the compiler is told about them. """ from __future__ import annotations @@ -22,7 +23,7 @@ # then -- for this package and for any tool introspecting an entity. import sys import types -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import is_dataclass from typing import ( TYPE_CHECKING, @@ -40,7 +41,7 @@ get_type_hints, ) -from typing_extensions import ReadOnly, is_typeddict +from typing_extensions import ReadOnly, TypeIs, is_typeddict from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET @@ -156,24 +157,45 @@ def is_optional(annotation: object) -> bool: return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) -def _no_nested_field(annotation: object) -> bool: - return False +class MetadataFieldValue: + """What a metadata field holds once read: an entity, or the JSON it could not read. + A base with no behaviour. A field annotated with a class deriving + from it -- `CodecEntity | Opaque` -- is a nested metadata field, which + is all the compiler needs to know of entities. + """ -nested_field: Callable[[object], bool] = _no_nested_field -"""Whether an annotation is a nested metadata field: an entity type, or a union of those with `Opaque`. + __slots__ = () -The one shape the compiler cannot recognise by itself, because which -classes are entities is `_entity`'s to say; it sets this once at import. -Consulted ahead of every other shape, since an entity is a dataclass too -and must not be walked as a record. -""" + +def unsubscripted(candidate: object) -> object: + """`CodecEntity[X]` as `CodecEntity`; anything else as it is.""" + origin = get_origin(candidate) + return origin if isinstance(origin, type) else candidate + + +def is_metadata_field_type(candidate: object) -> TypeIs[type[MetadataFieldValue]]: + """Whether `candidate` is a class a metadata field may hold a value of.""" + candidate = unsubscripted(candidate) + return isinstance(candidate, type) and issubclass(candidate, MetadataFieldValue) + + +def is_nested_field(annotation: object) -> bool: + """A metadata-field class, or a union of them (with `UNSET`, if optional).""" + candidates = [ + candidate + for candidate in (get_args(annotation) if is_union(annotation) else (annotation,)) + if candidate is not UNSET + ] + return len(candidates) != 0 and all( + is_metadata_field_type(candidate) for candidate in candidates + ) def describe(annotation: object) -> str: """The annotation as a message would name it: "an integer", "an object".""" inner, _ = strip_annotation(annotation) - if nested_field(inner): + if is_nested_field(inner): return "a metadata field" if inner is int: return "an integer" @@ -213,7 +235,7 @@ def shape_of(annotation: object) -> str | None: None means any shape -- a JSON value, or a union that mixes them. """ inner, _ = strip_annotation(annotation) - if nested_field(inner): + if is_nested_field(inner): return "field" if inner is int: return "int" @@ -402,7 +424,7 @@ def check_for(annotation: object) -> TypeCheck | None: those, a nested object described by a TypedDict or a record dataclass, an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested metadata field -- an entity type, with or - without `Opaque`, which `nested_field` recognises. `UNSET` in a union + without `Opaque`, which `is_nested_field` recognises. `UNSET` in a union says the member may be absent, which is the other half of a table entry and is read separately by `is_optional`. @@ -411,7 +433,7 @@ def check_for(annotation: object) -> TypeCheck | None: one of these shapes instead, with any finer rule in `__post_init__`. """ inner, _ = strip_annotation(annotation) - if nested_field(inner): + if is_nested_field(inner): return is_metadata_field if inner is int: return is_int @@ -513,6 +535,7 @@ def declared_class_vars(cls: type) -> dict[str, type]: __all__ = [ "FROM_NAME", + "MetadataFieldValue", "any_of", "check_for", "declared_class_vars", @@ -524,11 +547,13 @@ def declared_class_vars(cls: type) -> dict[str, type]: "fixed_tuple", "has_shape", "is_class_var", + "is_metadata_field_type", + "is_nested_field", "is_optional", "is_union", "mapping_of", - "nested_field", "own_annotations", "shape_of", "strip_annotation", + "unsubscripted", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 4ca5a769c9..fe02ea8ec3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -31,7 +31,7 @@ # (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class # creation, and a name that exists only for the type checker is a NameError # then -- for this package and for any tool introspecting an entity. -from collections.abc import Callable, Mapping, Sequence # noqa: TC003 +from collections.abc import Callable, Mapping # noqa: TC003 from copy import deepcopy from dataclasses import MISSING, Field, dataclass, is_dataclass, replace from types import MappingProxyType @@ -49,6 +49,7 @@ get_origin, get_type_hints, ) +from weakref import WeakKeyDictionary from typing_extensions import TypeIs, TypeVar, is_typeddict @@ -65,7 +66,6 @@ from zarr_metadata.v3._parts import ArrayParts from zarr_metadata.v3._registry import Context -from zarr_metadata.v3 import _compile from zarr_metadata.v3._checks import ( Loc, MemberTypes, @@ -86,6 +86,7 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._compile import ( FROM_NAME, + MetadataFieldValue, declared_class_vars, derive_member_types, element_annotations, @@ -93,10 +94,13 @@ field_hints, has_shape, is_class_var, + is_metadata_field_type, + is_nested_field, is_union, own_annotations, shape_of, strip_annotation, + unsubscripted, ) EntityT = TypeVar("EntityT", bound="MetadataEntity") @@ -199,17 +203,6 @@ def _is_entity_kind(candidate: object) -> TypeIs[type[MetadataEntity]]: return isinstance(candidate, type) and issubclass(candidate, MetadataEntity) -def _unsubscripted(candidate: object) -> object: - """`CodecEntity[X]` as `CodecEntity`; anything else as it is.""" - origin = get_origin(candidate) - return origin if isinstance(origin, type) else candidate - - -def _is_entity_type(candidate: object) -> bool: - candidate = _unsubscripted(candidate) - return candidate is Opaque or _is_entity_kind(candidate) - - def json_type_of(cls: type[MetadataEntity]) -> object: """The JSON type `cls` names for `to_json`; the default if it names none. @@ -228,30 +221,10 @@ def json_type_of(cls: type[MetadataEntity]) -> object: return ZarrV3MetadataFieldJSON -def _is_entity_or_opaque(candidates: Sequence[object]) -> bool: - """A nested metadata field: some entity kind, optionally with `Opaque`.""" - return ( - len(candidates) != 0 - and all(_is_entity_type(candidate) for candidate in candidates) - and any(candidate is not Opaque for candidate in candidates) - ) - - -def _is_nested_field(annotation: object) -> bool: - """An entity type, or a union of entity types and `Opaque`.""" - candidates = list(get_args(annotation)) if is_union(annotation) else [annotation] - return _is_entity_or_opaque([candidate for candidate in candidates if candidate is not UNSET]) - - -# The compiler knows nothing of entities; this is where it learns which -# annotations are nested metadata fields. -_compile.nested_field = _is_nested_field - - def contains_entity(annotation: object) -> bool: """Whether a value of this type holds a nested metadata field anywhere in it.""" inner, _ = strip_annotation(annotation) - if _is_entity_type(inner): + if is_metadata_field_type(inner): return True origin = get_origin(inner) if is_union(inner): @@ -274,7 +247,7 @@ def _as_entity_kind(candidate: object) -> type[MetadataEntity] | None: narrows this parameter and not the caller's variable, which the caller goes on to read as the annotation it is. """ - candidate = _unsubscripted(candidate) + candidate = unsubscripted(candidate) if _is_entity_kind(candidate): return candidate return None @@ -292,7 +265,7 @@ def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: return [kind for arg in arguments if arg is not UNSET for kind in _entity_kinds(arg)] if origin is tuple: return [kind for arg in arguments if arg is not Ellipsis for kind in _entity_kinds(arg)] - if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + if isinstance(inner, type) and is_dataclass(inner) and not is_metadata_field_type(inner): return [kind for value in field_hints(inner).values() for kind in _entity_kinds(value)] return [] @@ -326,15 +299,13 @@ def _resolve( ) -> tuple[object, tuple[ValidationProblem, ...]]: """`value`, with every nested metadata field in it read as an entity in `context`. - What `prepare` used to be written for by hand: a field annotated with - an entity type is resolved through the scope, at the point that kind - of entity is registered at; an array of them element by element; a - record holding one field by field. The value has passed its type - check, so the shapes are the annotation's. + A field annotated with an entity type is resolved through the scope, + at the point that kind of entity is registered at; an array of them + element by element; a record holding one field by field. The value + has passed its type check, so the shapes are the annotation's. """ inner, _ = strip_annotation(annotation) - candidates = list(get_args(inner)) if is_union(inner) else [inner] - if _is_entity_or_opaque(candidates): + if is_nested_field(inner): return context.coerce(_point_of(_entity_kinds(inner)[0]), value, loc) if is_union(inner): branch = _fitting_branch(inner, value) @@ -350,7 +321,7 @@ def _resolve( resolved.append(item) found.extend(problems) return tuple(resolved), tuple(found) - if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + if isinstance(inner, type) and is_dataclass(inner) and not is_metadata_field_type(inner): entries = cast("Mapping[str, object]", value) members: dict[str, object] = {} found = [] @@ -382,7 +353,7 @@ def render_nested(annotation: object, value: object) -> object: element_annotations(inner, len(entries)), entries, strict=True ) ) - if isinstance(inner, type) and is_dataclass(inner) and not _is_entity_type(inner): + if isinstance(inner, type) and is_dataclass(inner) and not is_metadata_field_type(inner): return { name: render_nested(field_annotation, getattr(value, name)) for name, field_annotation in field_hints(inner).items() @@ -412,7 +383,7 @@ def canonicalize_nested(annotation: object, value: object) -> object: if ( isinstance(inner, type) and is_dataclass(inner) - and not _is_entity_type(inner) + and not is_metadata_field_type(inner) and is_dataclass(value) and not isinstance(value, type) ): @@ -426,72 +397,80 @@ def canonicalize_nested(annotation: object, value: object) -> object: return value -_MISSING_DEFAULT: Final = object() -"""Distinguishes "declared no default" from a default that is None or UNSET.""" - - @dataclass(frozen=True, slots=True) -class Opaque: +class Opaque(MetadataFieldValue): """A metadata field this reading did not turn into an entity. - Carrying the JSON rather than dropping it is what makes the result a - real union: `CodecEntity | Opaque` is exhaustive and narrows, where - `CodecEntity | object` is just `object` and narrows to nothing. - - `reason` is the distinction a reader needs and could not otherwise - make. `out_of_scope` is a name no entity in this `Context` claims -- - an extension this reader does not model, which is not an error and is - the reader's cue to resolve it elsewhere. `invalid` is a name that - *was* claimed and then refused; the reasons are in the problems - reported alongside. + Carries the JSON the document wrote, so a reader holding a + `CodecEntity | Opaque` has everything the document said in either + case, and `reason` says which case it is. `out_of_scope` is a name no + entity in this `Context` claims -- an extension this reader does not + model, which is not an error and is the reader's cue to resolve it + elsewhere. `invalid` is a name that *was* claimed and then refused, + for the reasons reported alongside. """ json: object reason: Literal["out_of_scope", "invalid"] -# No `slots=True`, deliberately. It rebuilds the class, which on Python -# 3.11 and 3.12 leaves the zero-argument `super()` *in that same class's -# body* pointing at the class it replaced. Several entities call `super()` -# to narrow `to_json` and to adjust `configuration`, so they would each -# have to spell it `super(Cls, self)`. CPython fixed this in 3.13, so when -# that is the floor this is worth revisiting; the memory saved is small at -# document scale, which is why it has not been. -_DERIVED: Final = ("member_types", "configuration_required", "nested_members", "name_members") -"""The class variables `_compile_entity` derives; a declaration of one is refused.""" +@dataclass(frozen=True) +class Compiled: + """What the layer reads off an entity's fields, computed once per class. + + Kept beside the class rather than on it: a class is not written to + after its definition. + """ + + member_types: MemberTypes + """The configuration members, and the type check each one's annotation implies.""" + configuration_required: bool + """Whether some member is required, which decides whether the bare-name spelling is legal.""" + nested_members: Mapping[str, object] + """The fields that hold other entities, with their annotations.""" + name_members: tuple[str, ...] + """The fields the envelope's name carries, marked `Annotated[str, FROM_NAME]`.""" -def _compile_entity(cls: type[MetadataEntity]) -> None: - """Derive from the fields the tables the layer reads. +_COMPILED: Final[WeakKeyDictionary[type[MetadataEntity], Compiled]] = WeakKeyDictionary() - Raises for a declaration that cannot be compiled: a derived table - declared by hand, which the derivation would silently overwrite, and - a field annotation outside the shapes the compiler reads. + +def compiled(cls: type[MetadataEntity]) -> Compiled: + """The tables read off `cls`'s fields. + + Computed the first time a class is asked about and kept for its + lifetime. Raises `TypeError` for a field annotation outside the shapes + the compiler reads. """ - declared = [name for name in _DERIVED if name in vars(cls)] - if len(declared) != 0: - msg = ( - f"{cls.__name__} declares {', '.join(declared)}, which is derived from the " - "fields at class creation; remove the declaration" - ) - raise TypeError(msg) - cls.member_types, unread = derive_member_types(cls) - if len(unread) != 0: + found = _COMPILED.get(cls) + if found is None: + member_types, unread = derive_member_types(cls) + if len(unread) != 0: + hints = field_hints(cls) + msg = ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in sorted(unread))}" + ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in `__post_init__`" + ) + raise TypeError(msg) hints = field_hints(cls) - msg = ( - f"{cls.__name__}: {'; '.join(f'{name} is annotated {hints[name]!r}' for name in sorted(unread))}" - ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, a " - "Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass record, " - "Mapping[str, V], a NewType, or an entity kind with Opaque (CodecEntity | Opaque); " - "add | UNSET for an optional member, and put any finer rule in `__post_init__`" + found = _COMPILED[cls] = Compiled( + member_types=MappingProxyType(member_types), + configuration_required=any(required for required, _ in member_types.values()), + nested_members=MappingProxyType( + { + name: annotation + for name, annotation in hints.items() + if contains_entity(annotation) + } + ), + name_members=envelope_members(cls), ) - raise TypeError(msg) - cls.configuration_required = any(required for required, _ in cls.member_types.values()) - cls.name_members = envelope_members(cls) - hints = field_hints(cls) - cls.nested_members = { - name: annotation for name, annotation in hints.items() if contains_entity(annotation) - } + return found # The invariants, each a function of the compiled class returning why it @@ -521,7 +500,7 @@ def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: # field typed as one could not be resolved through a scope. unplaced = sorted( name - for name, annotation in cls.nested_members.items() + for name, annotation in compiled(cls).nested_members.items() if any(kind.extension_point is None for kind in _entity_kinds(annotation)) ) if len(unplaced) == 0: @@ -538,10 +517,10 @@ def _entity_unions_lacking_opaque(annotation: object) -> bool: inner, _ = strip_annotation(annotation) if is_union(inner): parts = [part for part in get_args(inner) if part is not UNSET] - if any(_is_entity_type(part) and part is not Opaque for part in parts): + if any(is_metadata_field_type(part) and part is not Opaque for part in parts): return Opaque not in parts return any(_entity_unions_lacking_opaque(part) for part in parts) - if _is_entity_type(inner): + if is_metadata_field_type(inner): return inner is not Opaque if get_origin(inner) is tuple: return any( @@ -558,7 +537,7 @@ def _nested_fields_admit_opaque(cls: type[MetadataEntity]) -> str | None: # `codec.inner.level` would be accepted and then raise. lacking = sorted( name - for name, annotation in cls.nested_members.items() + for name, annotation in compiled(cls).nested_members.items() if _entity_unions_lacking_opaque(annotation) ) if len(lacking) == 0: @@ -681,8 +660,8 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | f"{cls.__name__} names {json_type!r} as its JSON type, which is not an object " "TypedDict, a name type, or a union of one of each" ) - writes_bare = not cls.configuration_required and cls.must_understand - writes_object = len(cls.member_types) != 0 or not cls.must_understand + writes_bare = not compiled(cls).configuration_required and cls.must_understand + writes_object = len(compiled(cls).member_types) != 0 or not cls.must_understand found: list[str] = [] if writes_bare and len(names) == 0: found.append("lacks the bare name the entity writes when every member is absent") @@ -720,19 +699,19 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | ) if not cls.must_understand and "must_understand" not in hints: found.append("has no must_understand key, which the entity writes") - if len(cls.member_types) == 0: + if len(compiled(cls).member_types) == 0: if "configuration" in hints: found.append("has a configuration key, and the entity has no members") continue if "configuration" not in hints: found.append("has no configuration key, and the entity has members") continue - if ("configuration" in required) != cls.configuration_required: + if ("configuration" in required) != compiled(cls).configuration_required: found.append( "has configuration " + ("required" if "configuration" in required else "optional") + ", but a member is " - + ("required" if cls.configuration_required else "not required") + + ("required" if compiled(cls).configuration_required else "not required") ) configuration = hints["configuration"] if not is_typeddict(configuration): @@ -742,10 +721,10 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | except NameError: continue keys = configuration_hints.keys() - if set(keys) != set(cls.member_types): + if set(keys) != set(compiled(cls).member_types): found.append( f"has configuration keys {sorted(keys)!r} where the members are " - f"{sorted(cls.member_types)!r}" + f"{sorted(compiled(cls).member_types)!r}" ) continue configuration_required = { @@ -755,7 +734,7 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | } misstated = sorted( key - for key, (member_required, _) in cls.member_types.items() + for key, (member_required, _) in compiled(cls).member_types.items() if (key in configuration_required) != member_required ) if len(misstated) != 0: @@ -768,19 +747,19 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: - """Each member's declared default, or `_MISSING_DEFAULT`. + """Each member's declared default, or `MISSING`. `@dataclass` has not run yet -- `__init_subclass__` runs first -- so a member declared with `field(...)` is still a `Field` here and its default has to be unwrapped. """ defaulted: dict[str, object] = {} - for key in cls.member_types: - declared: object = getattr(cls, key, _MISSING_DEFAULT) + for key in compiled(cls).member_types: + declared: object = getattr(cls, key, MISSING) if type(declared) is Field: spec = cast("Field[object]", declared) declared = ( - _MISSING_DEFAULT + MISSING if spec.default is MISSING and spec.default_factory is MISSING else spec.default ) @@ -795,7 +774,7 @@ def _optional_members_default_to_unset(cls: type[MetadataEntity]) -> str | None: defaulted = _declared_defaults(cls) invented = [ key - for key, (required, _) in cls.member_types.items() + for key, (required, _) in compiled(cls).member_types.items() if not required and defaulted[key] is not UNSET ] if len(invented) == 0: @@ -816,8 +795,8 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: defaulted = _declared_defaults(cls) presumed = [ key - for key, (required, _) in cls.member_types.items() - if required and defaulted[key] is not _MISSING_DEFAULT + for key, (required, _) in compiled(cls).member_types.items() + if required and defaulted[key] is not MISSING ] if len(presumed) == 0: return None @@ -843,7 +822,7 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: @dataclass(frozen=True) -class MetadataEntity(Generic[JSONT_co]): +class MetadataEntity(MetadataFieldValue, Generic[JSONT_co]): """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what @@ -866,9 +845,9 @@ class MetadataEntity(Generic[JSONT_co]): and raises `MetadataValidationError` once -- so `BloscCodec(clevel=99)` raises, and `coerce` reports the same problems instead. `coerce`, `configuration`, `to_json` and `canonical` are written once here - against what the fields say, and the class variables below -- - `member_types`, `nested_members` -- are that reading, compiled at - class creation: nothing declares them. + against what the fields say; `compiled(cls)` is that reading, computed + once per class and kept beside it, so the class itself is never + written to. """ extension_point: ClassVar[ExtensionPointField | None] = None @@ -908,49 +887,12 @@ class creation: nothing declares them. an invented identifier that no real name can collide with. """ - member_types: ClassVar[MemberTypes] = MappingProxyType({}) - """The configuration members, and the type each one takes. - - Read off the dataclass fields at class creation: which members there - are, which may be absent (the type admits `UNSET`), and the check - each one's type implies. `coerce` reads a configuration against it - member by member, so one member that cannot be read costs that member - and not the rest. An annotation the compiler cannot read is refused - at class creation; the field is written as a shape JSON takes. The public - JSON TypedDict is held to the same keys by `tests/v3/test_entities.py`. - """ - - name_members: ClassVar[tuple[str, ...]] = () - """The fields the envelope's name carries, marked `Annotated[str, FROM_NAME]`. - - Read off the fields at class creation. `coerce` fills each with the - name the document wrote, so a family whose validity is in its name - -- the raw-bytes `r` types -- needs no reading of its own. - """ - - nested_members: ClassVar[Mapping[str, object]] = MappingProxyType({}) - """The fields that hold other entities, with their annotations. - - Read off the fields at class creation, like `member_types`. These are - the members `coerce` resolves through the scope, `configuration` - renders as JSON and `canonical` recurses into -- so an entity that - contains entities writes nothing for any of that. - """ - - configuration_required: ClassVar[bool] = False - """Whether the bare-name spelling says too little for this entity. - - The spec permits a bare name "if no configuration metadata is - required", so this is true exactly when some member is required -- - which the fields already say. - """ - def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: """Compile the entity from its fields, and refuse one this layer cannot use. - `_compile_entity` derives the tables the layer reads -- the - members and their checks, the nested fields, the value rules -- - and then every invariant in `_INVARIANTS` is asked. Each names + `compiled` reads the tables the layer needs off the fields -- the + members and their checks, the nested fields -- and then every + invariant in `_INVARIANTS` is asked. Each names something that type-checks cleanly and then goes wrong later, somewhere that will not name this class; an import-time error in the extension's own module is the one place the author is looking. @@ -963,9 +905,10 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: return if "__dataclass_fields__" in vars(cls): # `@dataclass(slots=True)` builds the class a second time from - # the first one's dict, tables included: compiled already. + # the first one's dict: verified already, and its members are + # slot descriptors now rather than the defaults the checks read. return - _compile_entity(cls) + compiled(cls) for invariant in _INVARIANTS: message = invariant(cls) if message is not None: @@ -991,21 +934,22 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: if name is None or not cls.accepts(name): return None, problem((), f"expected the {cls.identifier!r} entity") if configuration is None: - if cls.configuration_required: + if compiled(cls).configuration_required: return None, problem( ("configuration",), f"{cls.identifier!r} requires a configuration", "missing_key", ) configuration = cast("Mapping[str, object]", {}) - members, own = coerce_members(configuration, cls.member_types) - for member in cls.name_members: + spec = compiled(cls) + members, own = coerce_members(configuration, spec.member_types) + for member in spec.name_members: members[member] = name # A member that is itself an entity is read in the scope whatever # else was found: its problems are determinable, so they are # reported in the same pass. found = own - for member, annotation in cls.nested_members.items(): + for member, annotation in spec.nested_members.items(): if member in members: members[member], nested = _resolve( annotation, members[member], context, ("configuration", member) @@ -1050,7 +994,7 @@ def canonical(self) -> Self: the entity's own rewrite, which is where an entity says that two spellings of its own members mean the same. """ - nested = type(self).nested_members + nested = compiled(type(self)).nested_members walked = ( self if len(nested) == 0 @@ -1097,7 +1041,7 @@ def configuration(self) -> dict[str, object]: through the document it returned. """ members = self._configuration_members() - for name, annotation in type(self).nested_members.items(): + for name, annotation in compiled(type(self)).nested_members.items(): if name in members: members[name] = render_nested(annotation, members[name]) return deepcopy(members) @@ -1110,7 +1054,7 @@ def _configuration_members(self) -> dict[str, object]: """ return { key: value - for key in type(self).member_types + for key in compiled(type(self)).member_types if (value := getattr(self, key)) is not UNSET } @@ -1265,6 +1209,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "CodecEntity", "CodecKind", "Coerced", + "Compiled", "DataTypeEntity", "ExtensionPointField", "JSONT_co", @@ -1275,6 +1220,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "StorageClass", "TypeCheck", "coerce_members", + "compiled", "is_bool", "is_entity", "is_int", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index c87e93c149..144497b23f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -6,9 +6,9 @@ apart from one number -- the range, the hex parser, the component type -- so each family is written once and parameterised by that number. -The alternative, a table keyed by data type name, is what this replaces: -it put the knowledge of what `int32` accepts somewhere other than -`int32`, and needed a drift test to keep the two in step. +Written per family rather than kept in a table keyed by name, so that +what `int32` accepts is stated where `int32` is, with nothing elsewhere +to keep in step with it. """ from __future__ import annotations diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index f04e2a7880..65ca4a3bd6 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -347,22 +347,6 @@ class Structured(CodecEntity): kind: ClassVar[CodecKind] = "bytes_bytes" -@pytest.mark.parametrize( - "name", - ["member_types", "configuration_required", "nested_members"], -) -def test_error_a_derived_class_variable_may_not_be_declared(name: str) -> None: - # Each is read off the fields at class creation, and a declaration - # would be silently overwritten by that reading. Built with `type`, - # since a class body cannot spell a name from a parameter. - with pytest.raises(TypeError, match=f"declares {name}, which is derived from the fields"): - type( - "Opinionated", - (CodecEntity,), - {"identifier": "acme.opinionated", "kind": "bytes_bytes", name: {}}, - ) - - # A third-party codec that contains another codec: the case that used to # need `prepare`, `configuration` and `canonical` written by hand. @dataclass(frozen=True) @@ -555,7 +539,6 @@ class AcmeSlotted(CodecEntity): identifier: ClassVar[str] = "acme.slotted" kind: ClassVar[CodecKind] = "bytes_bytes" - assert list(AcmeSlotted.member_types) == ["level"] assert AcmeSlotted(level=1).to_json() == { "name": "acme.slotted", "configuration": {"level": 1}, @@ -569,7 +552,7 @@ class AcmeNoted(CodecEntity): kind: ClassVar[CodecKind] = "bytes_bytes" note: ClassVar = "not a member" - assert AcmeNoted.member_types == {} + assert AcmeNoted().to_json() == "acme.noted" def test_a_number_member_is_a_float_field() -> None: From a5f41a47d908078b5892c3645daba1eb9e255d33 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 15:07:23 +0200 Subject: [PATCH 078/107] refactor(zarr-metadata): what a kind must answer is abstract on it Review comment: the invariant that read `cls.transition is not CodecEntity.transition` was doing by hand what the language does. The entity base is an ABC. A codec's kind is its base class -- `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` -- and `transition` is abstract on the first; `fill_value_problems` is abstract on `DataTypeEntity` and `grid` on `ChunkGridEntity`, so a data type that accepts every fill value says so with `return ()` instead of inheriting a silent default. Registration refuses an entity that leaves a hook abstract, naming it. Two reflective invariants -- the one that resolved `Literal` class-variable annotations through a shell class, and the `transition` one -- are replaced by two explicit ones: a codec is of a kind class, and `kind` and `scalar_storage` hold listed values. The nine codecs subclass their kind and drop the `kind` line; the chain narrows with `isinstance(codec, ArrayArrayCodec)`; the door exports the kinds and its example is a `BytesBytesCodec`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 12 ++ .../src/zarr_metadata/v3/_chain.py | 8 +- .../src/zarr_metadata/v3/_entity.py | 142 ++++++++++-------- .../src/zarr_metadata/v3/_registry.py | 11 ++ .../src/zarr_metadata/v3/codec/blosc.py | 6 +- .../src/zarr_metadata/v3/codec/bytes.py | 6 +- .../src/zarr_metadata/v3/codec/cast_value.py | 6 +- .../src/zarr_metadata/v3/codec/crc32c.py | 6 +- .../src/zarr_metadata/v3/codec/gzip.py | 6 +- .../zarr_metadata/v3/codec/scale_offset.py | 6 +- .../v3/codec/sharding_indexed.py | 5 +- .../src/zarr_metadata/v3/codec/transpose.py | 6 +- .../src/zarr_metadata/v3/codec/zstd.py | 6 +- .../src/zarr_metadata/v3/entity.py | 55 ++++--- .../tests/v3/test_acme_affine.py | 6 +- .../tests/v3/test_extension_api.py | 103 +++++++------ 16 files changed, 213 insertions(+), 177 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index f85efed366..25dc36dc0f 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -68,6 +68,18 @@ hand-written check vocabulary the compiler made obsolete. The two reviewer-written extensions are kept as `tests/v3/test_acme_affine.py` and `tests/v3/test_acme_decimal.py`, written against the door alone. +From the review of that: the entity base is an ABC, and what a kind +must answer is abstract on it -- `transition` on `ArrayArrayCodec`, +`fill_value_problems` on `DataTypeEntity`, `grid` on `ChunkGridEntity` +-- so a codec's kind is its base class (`ArrayArrayCodec`, +`ArrayBytesCodec`, `BytesBytesCodec`) rather than a string it sets, and +an entity that leaves a hook undefined is refused at registration +rather than accepted with a silent default. The tables the layer reads +off an entity's fields live in `compiled(cls)`, a record kept beside the +class: a class is not written to after its definition. The compiler +recognises a nested metadata field by a marker base, `MetadataFieldValue`, +rather than by a module attribute set from outside. + One thing this does not change, under mypy. An entity's JSON type is a TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as `Mapping[str, object]`, never as the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py index 411b5c0e80..0b4bd39f03 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.v3._entity import CodecEntity, within +from zarr_metadata.v3._entity import ArrayArrayCodec, CodecEntity, within if TYPE_CHECKING: from collections.abc import Sequence @@ -102,9 +102,9 @@ def chain_problems( continue problems.extend(within((*loc, index), codec.incoming_problems(incoming))) incoming = ( - None - if incoming is None or type(codec).kind != "array_array" - else codec.transition(incoming) + codec.transition(incoming) + if incoming is not None and isinstance(codec, ArrayArrayCodec) + else None ) return tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index fe02ea8ec3..a42e81832b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -27,6 +27,8 @@ from __future__ import annotations +from abc import ABC, abstractmethod + # Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below # (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class # creation, and a name that exists only for the type checker is a NameError @@ -58,12 +60,11 @@ MetadataValidationError, ValidationProblem, ) -from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: from typing import Self - from zarr_metadata.v3._parts import ArrayParts + from zarr_metadata.v3._parts import ArrayParts, ChunkGrid from zarr_metadata.v3._registry import Context from zarr_metadata.v3._checks import ( @@ -583,49 +584,6 @@ def _owed_class_variables_are_declared(cls: type[MetadataEntity]) -> str | None: ) -def _literal_class_variables_hold_a_listed_value(cls: type[MetadataEntity]) -> str | None: - # A class variable typed as a `Literal` -- `kind`, `scalar_storage` -- - # is read by other entities' rules, which have nothing to say about a - # value outside the listed ones and would fall silent. - for name, annotating in declared_class_vars(cls).items(): - if not hasattr(cls, name): - continue - shell = type( - "_ClassVar", - (), - { - "__annotations__": {name: own_annotations(annotating)[name]}, - "__module__": annotating.__module__, - }, - ) - try: - hint = get_type_hints(shell)[name] - except NameError: - # Typed with a name imported only for the type checker. - continue - inner = get_args(hint)[0] if get_origin(hint) is ClassVar else hint - if get_origin(inner) is Literal and getattr(cls, name) not in get_args(inner): - return ( - f"{cls.__name__} sets {name} = {getattr(cls, name)!r}, " - f"which is not one of {get_args(inner)!r}" - ) - return None - - -def _array_array_codecs_define_transition(cls: type[MetadataEntity]) -> str | None: - # The default `transition` is None -- undeterminable -- which stops - # every rule after the codec. Right for a codec that could not say; - # wrong to accept silently from one being written now. - if not (issubclass(cls, CodecEntity) and cls.kind == "array_array"): - return None - if cls.transition is not CodecEntity.transition: - return None - return ( - f"{cls.__name__} is an array_array codec and does not define transition; return " - "incoming if it leaves the array's parts unchanged, or the parts it hands the next codec" - ) - - def _is_name_type(part: object) -> bool: """A JSON type for the bare-name spelling: `str`, a `Literal` of names, or a `NewType` of `str`.""" return ( @@ -746,6 +704,33 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | return f"{cls.__name__} names {json_type!r} as its JSON type, which " + "; ".join(found) +def _codecs_are_of_a_kind(cls: type[MetadataEntity]) -> str | None: + # The kind is the base class, and what a kind must answer is abstract + # on it; a codec that skips the kind classes skips that. + if not issubclass(cls, CodecEntity): + return None + if issubclass(cls, (ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec)): + return None + return ( + f"{cls.__name__} subclasses CodecEntity directly; subclass ArrayArrayCodec, " + "ArrayBytesCodec or BytesBytesCodec, which says what the codec does to the array" + ) + + +def _class_variables_hold_listed_values(cls: type[MetadataEntity]) -> str | None: + # Other entities' rules read these and have nothing to say about a + # value outside the listed ones: the endian rule would fall silent. + listed: tuple[tuple[str, tuple[object, ...]], ...] = () + if issubclass(cls, CodecEntity): + listed = (("kind", get_args(CodecKind)),) + elif issubclass(cls, DataTypeEntity): + listed = (("scalar_storage", get_args(StorageClass)),) + for name, values in listed: + if hasattr(cls, name) and getattr(cls, name) not in values: + return f"{cls.__name__} sets {name} = {getattr(cls, name)!r}, which is not one of {values!r}" + return None + + def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: """Each member's declared default, or `MISSING`. @@ -813,8 +798,8 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: _fields_do_not_shadow_class_variables, _owed_class_variables_are_declared, _named_json_type_matches_what_is_written, - _literal_class_variables_hold_a_listed_value, - _array_array_codecs_define_transition, + _codecs_are_of_a_kind, + _class_variables_hold_listed_values, _optional_members_default_to_unset, _required_members_have_no_default, ) @@ -822,7 +807,7 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: @dataclass(frozen=True) -class MetadataEntity(MetadataFieldValue, Generic[JSONT_co]): +class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what @@ -1103,12 +1088,18 @@ def to_json(self) -> JSONT_co: @dataclass(frozen=True) class CodecEntity(MetadataEntity[JSONT_co], base=True): - """An entity that occupies a position in the codec pipeline.""" + """An entity that occupies a position in the codec pipeline. + + Of one of three kinds, each a base class: `ArrayArrayCodec`, + `ArrayBytesCodec`, `BytesBytesCodec`. The kind fixes where in the + pipeline the codec may stand, and what it must answer. + """ extension_point: ClassVar[ExtensionPointField] = CODECS """Where a codec is registered, and so where a field typed as one is resolved.""" kind: ClassVar[CodecKind] + """Set by the kind class.""" variable_size: ClassVar[bool] = False """Whether this codec's output size depends on the bytes it is given. @@ -1122,24 +1113,41 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb `incoming` is None once the chain can no longer say what reaches here, and the default answer to that is nothing: declining beats - guessing. Locations are relative to this codec's `configuration`, - as `problems`' are; an empty one lands on the codec itself. + guessing. Locations are relative to this codec's `configuration`; + an empty one lands on the codec itself. """ return () - def transition(self, incoming: ArrayParts) -> ArrayParts | None: - """What the next codec in the chain sees, or None if undeterminable. - Only an array-to-array codec has anything to say: the two later - kinds end shape propagation by construction, one by consuming the - array and the other by never having had it. +@dataclass(frozen=True) +class ArrayArrayCodec(CodecEntity[JSONT_co], base=True): + """A codec that transforms the array: what reaches the next codec is its to say.""" + + kind: ClassVar[CodecKind] = "array_array" + + @abstractmethod + def transition(self, incoming: ArrayParts) -> ArrayParts | None: + """What the next codec in the chain sees. - The default is None, so a modelled codec that forgets to say how - it transforms the array stops propagation rather than silently - claiming to leave it alone. Failing closed here costs a judgment; - failing open would invent one. + `incoming` itself if this codec leaves the array's shape, grid and + data type alone; the parts it hands on if it changes one; None if + that cannot be determined from the metadata, which ends the + judgments downstream rather than inventing them. """ - return None + + +@dataclass(frozen=True) +class ArrayBytesCodec(CodecEntity[JSONT_co], base=True): + """The one codec in a pipeline that turns the array into bytes.""" + + kind: ClassVar[CodecKind] = "array_bytes" + + +@dataclass(frozen=True) +class BytesBytesCodec(CodecEntity[JSONT_co], base=True): + """A codec that transforms bytes, after the array is gone.""" + + kind: ClassVar[CodecKind] = "bytes_bytes" @dataclass(frozen=True) @@ -1156,6 +1164,7 @@ def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """ return () + @abstractmethod def grid(self, array_shape: object) -> ChunkGrid: """What this grid divides an array of `array_shape` into. @@ -1163,7 +1172,6 @@ def grid(self, array_shape: object) -> ChunkGrid: alone: a grid whose own metadata cannot be read still has the array's rank, and rank is enough for several rules. """ - return ChunkGrid.unreadable(array_shape) @dataclass(frozen=True) @@ -1188,14 +1196,13 @@ def storage_class(self) -> StorageClass | None: """ return type(self).scalar_storage + @abstractmethod def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """Why `value` is not a fill value of this type, if it is not. - Default: nothing. A data type this package does not model accepts - whatever its extension says it does, and guessing would reject - valid documents. + Every data type answers this; one that accepts any fill value + says so with `return ()`. """ - return () __all__ = [ @@ -1205,6 +1212,9 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "DATA_TYPE", "FROM_NAME", "STORAGE_TRANSFORMERS", + "ArrayArrayCodec", + "ArrayBytesCodec", + "BytesBytesCodec", "ChunkGridEntity", "CodecEntity", "CodecKind", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 1932215cef..80b73c9b5b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -21,6 +21,7 @@ from __future__ import annotations +import inspect from dataclasses import dataclass from typing import TYPE_CHECKING, Final, Literal, overload @@ -188,6 +189,16 @@ def __post_init__(self) -> None: f"{entity.__name__}.identifier" ) raise ValueError(msg) + if inspect.isabstract(entity): + # What a kind leaves abstract -- `transition`, `grid`, + # `fill_value_problems` -- the entity answers, or it + # is not one this scope can use. + left = ", ".join(sorted(entity.__abstractmethods__)) + msg = ( + f"{entity.__name__} does not define {left}, which its base leaves " + "abstract; define it, if only to return the same thing as `incoming` or ()" + ) + raise TypeError(msg) if "__dataclass_fields__" not in vars(entity) and any( not is_class_var(annotation) for annotation in own_annotations(entity).values() ): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index ccbbd176ad..15a2be84ee 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -12,8 +12,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + BytesBytesCodec, problem, ) @@ -89,7 +88,7 @@ class BloscCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BloscCodec(CodecEntity[BloscCodecMetadata]): +class BloscCodec(BytesBytesCodec[BloscCodecMetadata]): """The `blosc` codec, coerced from its metadata. Everything blosc knows about itself: the shape its metadata takes, the @@ -105,7 +104,6 @@ class BloscCodec(CodecEntity[BloscCodecMetadata]): identifier: ClassVar[str] = BLOSC_CODEC_NAME variable_size: ClassVar[bool] = True - kind: ClassVar[CodecKind] = "bytes_bytes" # Every member is required but `typesize`, which only means something # when shuffling; `problems` is where that conditional lives. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index bb8ddb9dff..9928a9f6f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -12,8 +12,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + ArrayBytesCodec, DataTypeEntity, problem, ) @@ -80,7 +79,7 @@ class BytesCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BytesCodec(CodecEntity[BytesCodecMetadata]): +class BytesCodec(ArrayBytesCodec[BytesCodecMetadata]): """The `bytes` codec, coerced from its metadata. `endian` is optional and absent means something: a one-byte data type @@ -90,7 +89,6 @@ class BytesCodec(CodecEntity[BytesCodecMetadata]): endian: Endianness | UNSET = UNSET identifier: ClassVar[str] = BYTES_CODEC_NAME - kind: ClassVar[CodecKind] = "array_bytes" def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """The data type reaching here must have a raw byte representation. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index ad22f71219..fa60991fb2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -13,8 +13,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + ArrayArrayCodec, DataTypeEntity, Opaque, ) @@ -126,7 +125,7 @@ class CastValueCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class CastValueCodec(CodecEntity[CastValueCodecMetadata]): +class CastValueCodec(ArrayArrayCodec[CastValueCodecMetadata]): """The `cast_value` codec, coerced from its metadata. Holds the data type it casts to, so like `sharding_indexed` it is @@ -139,7 +138,6 @@ class CastValueCodec(CodecEntity[CastValueCodecMetadata]): scalar_map: ScalarMap | UNSET = UNSET identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME - kind: ClassVar[CodecKind] = "array_array" def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 454ff94b35..6bfce5e188 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -13,8 +13,7 @@ from typing_extensions import TypedDict from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + BytesBytesCodec, ) CRC32C_CODEC_NAME: Final = "crc32c" @@ -61,11 +60,10 @@ class Crc32cCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class Crc32cCodec(CodecEntity[Crc32cCodecName]): +class Crc32cCodec(BytesBytesCodec[Crc32cCodecName]): """The `crc32c` codec, coerced from its metadata. The name says everything: a checksum has nothing to configure. """ identifier: ClassVar[str] = CRC32C_CODEC_NAME - kind: ClassVar[CodecKind] = "bytes_bytes" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 9774f298b7..a90ffecf29 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -11,8 +11,7 @@ from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + BytesBytesCodec, problem, ) @@ -67,14 +66,13 @@ class GzipCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class GzipCodec(CodecEntity[GzipCodecMetadata]): +class GzipCodec(BytesBytesCodec[GzipCodecMetadata]): """The `gzip` codec, coerced from its metadata.""" level: int identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True - kind: ClassVar[CodecKind] = "bytes_bytes" def __post_init__(self) -> None: if not 0 <= self.level <= 9: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index f3d9f4d359..b3295fd458 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -13,8 +13,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + ArrayArrayCodec, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -74,7 +73,7 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ScaleOffsetCodec(CodecEntity[ScaleOffsetCodecMetadata]): +class ScaleOffsetCodec(ArrayArrayCodec[ScaleOffsetCodecMetadata]): """The `scale_offset` codec, coerced from its metadata. Both members are optional and any JSON scalar is well-typed here; what @@ -86,7 +85,6 @@ class ScaleOffsetCodec(CodecEntity[ScaleOffsetCodecMetadata]): scale: JSONValue | UNSET = UNSET identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME - kind: ClassVar[CodecKind] = "array_array" def __post_init__(self) -> None: """Each value is a scalar of the array's type, so neither is null. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 4667a4c691..d59165bdc0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -14,8 +14,8 @@ from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( + ArrayBytesCodec, CodecEntity, - CodecKind, Opaque, problem, ) @@ -95,7 +95,7 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ShardingIndexedCodec(CodecEntity[ShardingIndexedCodecMetadata]): +class ShardingIndexedCodec(ArrayBytesCodec[ShardingIndexedCodecMetadata]): """The `sharding_indexed` codec, coerced from its metadata. Holds two codec pipelines, so it is one of the few entities that @@ -110,7 +110,6 @@ class ShardingIndexedCodec(CodecEntity[ShardingIndexedCodecMetadata]): identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME variable_size: ClassVar[bool] = True - kind: ClassVar[CodecKind] = "array_bytes" def __post_init__(self) -> None: found: list[ValidationProblem] = [] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 3d8edb3a4e..db74338e61 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -11,8 +11,7 @@ from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + ArrayArrayCodec, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -63,13 +62,12 @@ class TransposeCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class TransposeCodec(CodecEntity[TransposeCodecMetadata]): +class TransposeCodec(ArrayArrayCodec[TransposeCodecMetadata]): """The `transpose` codec, coerced from its metadata.""" order: tuple[int, ...] identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME - kind: ClassVar[CodecKind] = "array_array" def __post_init__(self) -> None: """`order` must permute its own axes. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index b853475973..e8dde0501c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -14,8 +14,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( - CodecEntity, - CodecKind, + BytesBytesCodec, problem, ) @@ -75,7 +74,7 @@ class ZstdCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ZstdCodec(CodecEntity[ZstdCodecMetadata]): +class ZstdCodec(BytesBytesCodec[ZstdCodecMetadata]): """The `zstd` codec, coerced from its metadata.""" level: int @@ -83,7 +82,6 @@ class ZstdCodec(CodecEntity[ZstdCodecMetadata]): identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True - kind: ClassVar[CodecKind] = "bytes_bytes" def __post_init__(self) -> None: if not ZSTD_MIN_LEVEL <= self.level <= ZSTD_MAX_LEVEL: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 03f450f4d1..3c90a31f75 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -36,10 +36,12 @@ `Coerced`. Constructing an entity by hand raises `MetadataValidationError` with `loc` relative to the configuration: `("level",)`. -**Writing an extension.** Subclass `CodecEntity`, `DataTypeEntity`, -`ChunkGridEntity` or `MetadataEntity`; declare the configuration as -dataclass fields; put every rule finer than a type in `__post_init__`; -add the class to a scope. Complete, and runnable as written: +**Writing an extension.** Subclass the kind of thing it is -- a codec's +kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), +`DataTypeEntity`, `ChunkGridEntity`, or `MetadataEntity` for the two +points that take anything; declare the configuration as dataclass +fields; put every rule finer than a type in `__post_init__`; add the +class to a scope. Complete, and runnable as written: from dataclasses import dataclass from typing import ClassVar @@ -48,18 +50,16 @@ from zarr_metadata.v3.entity import ( CORE_AND_EXTENSIONS, UNSET, - CodecEntity, - CodecKind, + BytesBytesCodec, MetadataValidationError, problem, ) @dataclass(frozen=True) # load-bearing: `coerce` builds the entity with cls(**members) - class AcmeLz4Codec(CodecEntity): + class AcmeLz4Codec(BytesBytesCodec): acceleration: int | UNSET = UNSET # optional: defaults to UNSET, never to a value identifier: ClassVar[str] = "acme.lz4" - kind: ClassVar[CodecKind] = "bytes_bytes" def __post_init__(self) -> None: if self.acceleration is not UNSET and not 1 <= self.acceleration <= 65537: @@ -107,30 +107,29 @@ def __post_init__(self) -> None: -- one class for every `acme.fixedN` -- overrides `accepts(name)` and keeps the name in a field marked `Annotated[str, FROM_NAME]`, which `coerce` fills from the envelope. -- A codec: `kind`. An `array_array` codec must define - `transition(incoming: ArrayParts) -> ArrayParts | None` -- return - `incoming` if it leaves the array's shape, grid and data type alone, - or the parts it hands the next codec -- and may define +- A codec: its kind is its base class. An `ArrayArrayCodec` defines + `transition(incoming: ArrayParts) -> ArrayParts | None` -- abstract: + return `incoming` if it leaves the array's shape, grid and data type + alone, or the parts it hands the next codec -- and any codec may define `incoming_problems(incoming)` for what it cannot take. `variable_size` - says its output length is not fixed. A `bytes_bytes` or `array_bytes` - codec defines neither. + says its output length is not fixed. - A data type: `scalar_storage`, one of `StorageClass` (the `bytes` codec asks it whether an endianness is needed), and - `fill_value_problems(value, loc)`, which judges a document's - `fill_value`; left undefined, every fill value is accepted. The - families `IntegerDataType`, `FloatDataType`, `ComplexDataType` and - `NumpyTimeDataType` carry those for the types they cover; a family of + `fill_value_problems(value, loc)`, abstract: it judges a document's + `fill_value`, and a type that accepts any says so with `return ()`. + The families `IntegerDataType`, `FloatDataType`, `ComplexDataType` and + `NumpyTimeDataType` carry both for the types they cover; a family of your own is a subclass declared with `base=True`, which owes nothing itself and passes its class variables down. -- A chunk grid: `grid(array_shape)` and `shape_problems`; see +- A chunk grid: `grid(array_shape)`, abstract, and `shape_problems`; see `ChunkGridEntity`. -The defaults fail closed for the package's own sake, so the ones an -author would otherwise miss -- an `array_array` codec without a -`transition`, a data type with a `scalar_storage` outside the listed -values, a nested field without `Opaque`, a class without `@dataclass` -(caught at registration, the first place that can see it) -- are refused -with a message that says what to write. +What a kind leaves abstract, registration refuses an entity for not +defining; the other mistakes an author would not otherwise see -- a +`scalar_storage` outside the listed values, a nested field without +`Opaque`, a class without `@dataclass`, a codec subclassing `CodecEntity` +instead of a kind -- are refused at class creation or registration with +a message that says what to write. **Naming the JSON type.** `CodecEntity[AcmeLz4Metadata]` types `to_json` as your own TypedDict rather than as any metadata field. The shape is @@ -180,6 +179,9 @@ def __post_init__(self) -> None: DATA_TYPE, FROM_NAME, STORAGE_TRANSFORMERS, + ArrayArrayCodec, + ArrayBytesCodec, + BytesBytesCodec, ChunkGridEntity, CodecEntity, CodecKind, @@ -214,8 +216,11 @@ def __post_init__(self) -> None: "FROM_NAME", "STORAGE_TRANSFORMERS", "UNSET", + "ArrayArrayCodec", + "ArrayBytesCodec", "ArrayDocumentV3", "ArrayParts", + "BytesBytesCodec", "ChunkGrid", "ChunkGridEntity", "CodecEntity", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index b59c61ebb9..9eed647658 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -23,10 +23,9 @@ from zarr_metadata.v3.entity import ( CORE_AND_EXTENSIONS, UNSET, + ArrayArrayCodec, ArrayDocumentV3, ArrayParts, - CodecEntity, - CodecKind, DataTypeEntity, MetadataValidationError, Opaque, @@ -49,7 +48,7 @@ class AcmeAffineObject(TypedDict, closed=True): @dataclass(frozen=True) -class AcmeAffineCodec(CodecEntity[AcmeAffineObject]): +class AcmeAffineCodec(ArrayArrayCodec[AcmeAffineObject]): """`x * scale + offset`, stored as `dtype` if one is named.""" scale: float @@ -57,7 +56,6 @@ class AcmeAffineCodec(CodecEntity[AcmeAffineObject]): dtype: DataTypeEntity | Opaque | UNSET = UNSET identifier: ClassVar[str] = "acme.affine" - kind: ClassVar[CodecKind] = "array_array" def __post_init__(self) -> None: found: list[ValidationProblem] = [] diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 65ca4a3bd6..27001ee5bf 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -24,17 +24,21 @@ CORE, CORE_AND_EXTENSIONS, FROM_NAME, + ArrayArrayCodec, ArrayDocumentV3, ArrayParts, + BytesBytesCodec, ChunkGridEntity, CodecEntity, CodecKind, Context, DataTypeEntity, IntegerDataType, + Loc, MetadataEntity, Opaque, StorageClass, + ValidationProblem, ZarrV3MetadataFieldJSON, problem, ) @@ -43,13 +47,12 @@ @dataclass(frozen=True) -class AcmeLz4Codec(CodecEntity): +class AcmeLz4Codec(BytesBytesCodec): """A third-party compressor.""" acceleration: int | UNSET = UNSET identifier: ClassVar[str] = "acme.lz4" - kind: ClassVar[CodecKind] = "bytes_bytes" variable_size: ClassVar[bool] = True def __post_init__(self) -> None: @@ -70,6 +73,9 @@ class AcmeFloat8DataType(DataTypeEntity): identifier: ClassVar[str] = "acme.float8" scalar_storage: ClassVar[StorageClass] = "single_byte" + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + return () + def _scope() -> Context: return CORE_AND_EXTENSIONS.extended_with( @@ -150,8 +156,8 @@ def test_error_an_entity_must_say_what_it_is() -> None: # Never bound: the guard raises while the class is being created, # which is the whole point -- so pyright cannot see it used. @dataclass(frozen=True) - class Nameless(CodecEntity): - kind: ClassVar[CodecKind] = "bytes_bytes" + class Nameless(BytesBytesCodec): + """A codec that forgot to say what it is.""" def test_error_a_registry_key_must_be_the_identifier() -> None: @@ -196,13 +202,12 @@ def test_error_an_optional_member_defaults_to_unset() -> None: with pytest.raises(TypeError, match="a default other than UNSET"): @dataclass(frozen=True) - class Inventive(CodecEntity): + class Inventive(BytesBytesCodec): # Optional by its type, so the annotation and the default agree # on that much; it is the default's value that is wrong. level: int | UNSET = 3 identifier: ClassVar[str] = "acme.inventive" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_a_reader_gets_entities_or_an_exception() -> None: @@ -273,11 +278,10 @@ def test_error_a_field_may_not_shadow_a_class_variable() -> None: with pytest.raises(TypeError, match="shadowing a class variable"): @dataclass(frozen=True) - class Negotiable(CodecEntity): + class Negotiable(BytesBytesCodec): must_understand: bool = True # pyright: ignore[reportIncompatibleVariableOverride] identifier: ClassVar[str] = "acme.negotiable" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: @@ -312,6 +316,9 @@ def accepts(cls, name: str) -> bool: def to_json(self) -> ZarrV3MetadataFieldJSON: return cast("ZarrV3MetadataFieldJSON", self.data_type_name) + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: + return () + def test_a_third_party_can_register_a_family() -> None: # One class for an unbounded set of names. Nothing in the package @@ -340,23 +347,21 @@ def test_error_a_member_needs_a_check_from_somewhere() -> None: with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): @dataclass(frozen=True) - class Structured(CodecEntity): + class Structured(BytesBytesCodec): inner: object identifier: ClassVar[str] = "acme.structured" - kind: ClassVar[CodecKind] = "bytes_bytes" # A third-party codec that contains another codec: the case that used to # need `prepare`, `configuration` and `canonical` written by hand. @dataclass(frozen=True) -class AcmeWrapperCodec(CodecEntity): +class AcmeWrapperCodec(BytesBytesCodec): """A codec that applies another codec after its own step.""" inner: CodecEntity | Opaque identifier: ClassVar[str] = "acme.wrapper" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> None: @@ -425,9 +430,8 @@ def test_error_an_entity_may_not_override_canonical() -> None: with pytest.raises(TypeError, match="put the entity's own rewrite in `simplified`"): @dataclass(frozen=True) - class Rewriter(CodecEntity): + class Rewriter(BytesBytesCodec): identifier: ClassVar[str] = "acme.rewriter" - kind: ClassVar[CodecKind] = "bytes_bytes" def canonical(self) -> Self: # pyright: ignore[reportIncompatibleMethodOverride] return self @@ -439,12 +443,11 @@ def test_simplified_composes_with_the_walk_into_contained_entities() -> None: # that `noshuffle` ignores, and the frame of 0 that means "unframed" # is dropped -- with nothing to call `super()` for. @dataclass(frozen=True) - class AcmeFramedCodec(CodecEntity): + class AcmeFramedCodec(BytesBytesCodec): inner: CodecEntity | Opaque frame: int | UNSET = UNSET identifier: ClassVar[str] = "acme.framed" - kind: ClassVar[CodecKind] = "bytes_bytes" def simplified(self) -> Self: return self if self.frame != 0 else replace(self, frame=UNSET) @@ -461,11 +464,10 @@ def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: with pytest.raises(TypeError, match="has no `extension_point`"): @dataclass(frozen=True) - class Vague(CodecEntity): + class Vague(BytesBytesCodec): inner: MetadataEntity | Opaque identifier: ClassVar[str] = "acme.vague" - kind: ClassVar[CodecKind] = "bytes_bytes" # The JSON types third-party entities name, at module level so their @@ -492,13 +494,12 @@ class AcmeBlockObject(TypedDict, closed=True): # A third-party rule about a member, written in `__post_init__`. @dataclass(frozen=True) -class AcmeBlockCodec(CodecEntity): +class AcmeBlockCodec(BytesBytesCodec): """A codec whose block size must be a power of two.""" block: int identifier: ClassVar[str] = "acme.block" - kind: ClassVar[CodecKind] = "bytes_bytes" def __post_init__(self) -> None: if self.block < 1 or self.block & (self.block - 1) != 0: @@ -533,11 +534,10 @@ def test_a_slotted_entity_is_compiled_once() -> None: # arrives with the derived tables already on it and must not be # refused as having declared them. @dataclass(frozen=True, slots=True) - class AcmeSlotted(CodecEntity): + class AcmeSlotted(BytesBytesCodec): level: int identifier: ClassVar[str] = "acme.slotted" - kind: ClassVar[CodecKind] = "bytes_bytes" assert AcmeSlotted(level=1).to_json() == { "name": "acme.slotted", @@ -547,9 +547,8 @@ class AcmeSlotted(CodecEntity): def test_a_bare_class_var_is_a_class_variable() -> None: @dataclass(frozen=True) - class AcmeNoted(CodecEntity): + class AcmeNoted(BytesBytesCodec): identifier: ClassVar[str] = "acme.noted" - kind: ClassVar[CodecKind] = "bytes_bytes" note: ClassVar = "not a member" assert AcmeNoted().to_json() == "acme.noted" @@ -560,11 +559,10 @@ def test_a_number_member_is_a_float_field() -> None: # point and refuses a bool, which is what a document's `2` and `true` # deserve. @dataclass(frozen=True) - class AcmeScaled(CodecEntity): + class AcmeScaled(ArrayArrayCodec): scale: float identifier: ClassVar[str] = "acme.scaled" - kind: ClassVar[CodecKind] = "array_array" def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming @@ -586,11 +584,10 @@ def test_error_an_entity_must_be_a_dataclass() -> None: # Class creation runs before `@dataclass` and cannot see it missing; # registration can, and says so instead of the first `coerce` failing # with the base class's `__init__`. - class Undecorated(CodecEntity): + class Undecorated(BytesBytesCodec): level: int identifier: ClassVar[str] = "acme.undecorated" - kind: ClassVar[CodecKind] = "bytes_bytes" with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): CORE_AND_EXTENSIONS.extended_with(codecs={Undecorated.identifier: Undecorated}) @@ -601,21 +598,45 @@ def test_error_a_nested_field_admits_opaque() -> None: with pytest.raises(TypeError, match="inner holds an entity but does not admit Opaque"): @dataclass(frozen=True) - class Closed(CodecEntity): + class Closed(BytesBytesCodec): inner: CodecEntity identifier: ClassVar[str] = "acme.closed" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_error_an_array_array_codec_defines_transition() -> None: - # Left at the default, every rule after the codec would go silent. - with pytest.raises(TypeError, match="array_array codec and does not define transition"): + # `transition` is abstract on the kind; a codec that leaves it so is + # refused where it is first used, with what to write. + @dataclass(frozen=True) + class Silent(ArrayArrayCodec): + identifier: ClassVar[str] = "acme.silent" + + with pytest.raises( + TypeError, match="does not define transition, which its base leaves abstract" + ): + CORE_AND_EXTENSIONS.extended_with(codecs={Silent.identifier: Silent}) + + +def test_error_a_codec_is_of_a_kind() -> None: + with pytest.raises( + TypeError, match="subclasses CodecEntity directly; subclass ArrayArrayCodec" + ): @dataclass(frozen=True) - class Silent(CodecEntity): - identifier: ClassVar[str] = "acme.silent" - kind: ClassVar[CodecKind] = "array_array" + class Kindless(CodecEntity): + identifier: ClassVar[str] = "acme.kindless" + kind: ClassVar[CodecKind] = "bytes_bytes" + + +def test_error_a_data_type_judges_its_fill_values() -> None: + # Abstract, so that accepting every fill value is said, not defaulted. + @dataclass(frozen=True) + class Lax(DataTypeEntity): + identifier: ClassVar[str] = "acme.lax" + scalar_storage: ClassVar[StorageClass] = "single_byte" + + with pytest.raises(TypeError, match="does not define fill_value_problems"): + CORE_AND_EXTENSIONS.extended_with(data_type={Lax.identifier: Lax}) def test_error_a_literal_class_variable_holds_a_listed_value() -> None: @@ -646,11 +667,10 @@ def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: ): @dataclass(frozen=True) - class Misnamed(CodecEntity[Literal["acme.misnamed"]]): + class Misnamed(BytesBytesCodec[Literal["acme.misnamed"]]): level: int identifier: ClassVar[str] = "acme.misnamed" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_error_the_named_json_type_must_name_what_the_entity_accepts() -> None: @@ -659,11 +679,10 @@ def test_error_the_named_json_type_must_name_what_the_entity_accepts() -> None: with pytest.raises(TypeError, match="names 'gzip', which the entity does not accept"): @dataclass(frozen=True) - class Impostor(CodecEntity[GzipCodecObject]): + class Impostor(BytesBytesCodec[GzipCodecObject]): level: int identifier: ClassVar[str] = "acme.impostor" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_error_the_named_json_type_must_have_the_members_as_keys() -> None: @@ -672,11 +691,10 @@ def test_error_the_named_json_type_must_have_the_members_as_keys() -> None: ): @dataclass(frozen=True) - class Mismatched(CodecEntity[AcmeLvlObject]): + class Mismatched(BytesBytesCodec[AcmeLvlObject]): level: int identifier: ClassVar[str] = "acme.lvl" - kind: ClassVar[CodecKind] = "bytes_bytes" def test_a_third_party_entity_may_name_its_json_type_or_not() -> None: @@ -684,11 +702,10 @@ def test_a_third_party_entity_may_name_its_json_type_or_not() -> None: # the entity's own type, held to the members at class creation -- and # either way the same dict comes back. @dataclass(frozen=True) - class AcmeTypedBlockCodec(CodecEntity[AcmeBlockObject]): + class AcmeTypedBlockCodec(BytesBytesCodec[AcmeBlockObject]): block: int identifier: ClassVar[str] = "acme.block" - kind: ClassVar[CodecKind] = "bytes_bytes" assert AcmeTypedBlockCodec(block=8).to_json() == { "name": "acme.block", From 9ce8d007347956810f003628a6157eb9ca2e3f33 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 15:22:51 +0200 Subject: [PATCH 079/107] refactor(zarr-metadata): the tables are cached reads of the fields, not a record The `Compiled` record, its weak side table and the `compiled()` accessor were memoization dressed up as architecture. What the layer reads off an entity's fields -- `member_types`, `nested_members`, `name_members` -- is three functions of the class under `functools.cache`, and `configuration_required` is a one-line function over the first. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_entity.py | 165 ++++++++---------- 1 file changed, 74 insertions(+), 91 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index a42e81832b..fc9dc740cc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -36,6 +36,7 @@ from collections.abc import Callable, Mapping # noqa: TC003 from copy import deepcopy from dataclasses import MISSING, Field, dataclass, is_dataclass, replace +from functools import cache from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -51,7 +52,6 @@ get_origin, get_type_hints, ) -from weakref import WeakKeyDictionary from typing_extensions import TypeIs, TypeVar, is_typeddict @@ -415,65 +415,6 @@ class Opaque(MetadataFieldValue): reason: Literal["out_of_scope", "invalid"] -@dataclass(frozen=True) -class Compiled: - """What the layer reads off an entity's fields, computed once per class. - - Kept beside the class rather than on it: a class is not written to - after its definition. - """ - - member_types: MemberTypes - """The configuration members, and the type check each one's annotation implies.""" - configuration_required: bool - """Whether some member is required, which decides whether the bare-name spelling is legal.""" - nested_members: Mapping[str, object] - """The fields that hold other entities, with their annotations.""" - name_members: tuple[str, ...] - """The fields the envelope's name carries, marked `Annotated[str, FROM_NAME]`.""" - - -_COMPILED: Final[WeakKeyDictionary[type[MetadataEntity], Compiled]] = WeakKeyDictionary() - - -def compiled(cls: type[MetadataEntity]) -> Compiled: - """The tables read off `cls`'s fields. - - Computed the first time a class is asked about and kept for its - lifetime. Raises `TypeError` for a field annotation outside the shapes - the compiler reads. - """ - found = _COMPILED.get(cls) - if found is None: - member_types, unread = derive_member_types(cls) - if len(unread) != 0: - hints = field_hints(cls) - msg = ( - f"{cls.__name__}: " - f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in sorted(unread))}" - ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " - "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " - "record, Mapping[str, V], a NewType, or an entity kind with Opaque " - "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in `__post_init__`" - ) - raise TypeError(msg) - hints = field_hints(cls) - found = _COMPILED[cls] = Compiled( - member_types=MappingProxyType(member_types), - configuration_required=any(required for required, _ in member_types.values()), - nested_members=MappingProxyType( - { - name: annotation - for name, annotation in hints.items() - if contains_entity(annotation) - } - ), - name_members=envelope_members(cls), - ) - return found - - # The invariants, each a function of the compiled class returning why it # is refused, or None. Every one names something that type-checks cleanly # and then goes wrong somewhere that will not name the class. @@ -487,6 +428,52 @@ def compiled(cls: type[MetadataEntity]) -> Compiled: """What to do instead, for each method the base marks `@final`.""" +@cache +def member_types(cls: type[MetadataEntity]) -> MemberTypes: + """The configuration members, and the type check each one's annotation implies. + + Read off the fields. Raises `TypeError` for a field annotation outside + the shapes the compiler reads. + """ + derived, unread = derive_member_types(cls) + if len(unread) != 0: + hints = field_hints(cls) + msg = ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in sorted(unread))}" + ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in `__post_init__`" + ) + raise TypeError(msg) + return MappingProxyType(derived) + + +def configuration_required(cls: type[MetadataEntity]) -> bool: + """Whether some member is required, which is what makes the bare-name spelling illegal.""" + return any(required for required, _ in member_types(cls).values()) + + +@cache +def nested_members(cls: type[MetadataEntity]) -> Mapping[str, object]: + """The fields that hold other entities, with their annotations.""" + return MappingProxyType( + { + name: annotation + for name, annotation in field_hints(cls).items() + if contains_entity(annotation) + } + ) + + +@cache +def name_members(cls: type[MetadataEntity]) -> tuple[str, ...]: + """The fields the envelope's name carries, marked `Annotated[str, FROM_NAME]`.""" + return envelope_members(cls) + + def _final_methods_are_not_overridden(cls: type[MetadataEntity]) -> str | None: # `@final` is a promise pyright checks in the author's editor; this # is the same promise for a class created without one. @@ -501,7 +488,7 @@ def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: # field typed as one could not be resolved through a scope. unplaced = sorted( name - for name, annotation in compiled(cls).nested_members.items() + for name, annotation in nested_members(cls).items() if any(kind.extension_point is None for kind in _entity_kinds(annotation)) ) if len(unplaced) == 0: @@ -538,7 +525,7 @@ def _nested_fields_admit_opaque(cls: type[MetadataEntity]) -> str | None: # `codec.inner.level` would be accepted and then raise. lacking = sorted( name - for name, annotation in compiled(cls).nested_members.items() + for name, annotation in nested_members(cls).items() if _entity_unions_lacking_opaque(annotation) ) if len(lacking) == 0: @@ -618,8 +605,8 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | f"{cls.__name__} names {json_type!r} as its JSON type, which is not an object " "TypedDict, a name type, or a union of one of each" ) - writes_bare = not compiled(cls).configuration_required and cls.must_understand - writes_object = len(compiled(cls).member_types) != 0 or not cls.must_understand + writes_bare = not configuration_required(cls) and cls.must_understand + writes_object = len(member_types(cls)) != 0 or not cls.must_understand found: list[str] = [] if writes_bare and len(names) == 0: found.append("lacks the bare name the entity writes when every member is absent") @@ -657,19 +644,19 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | ) if not cls.must_understand and "must_understand" not in hints: found.append("has no must_understand key, which the entity writes") - if len(compiled(cls).member_types) == 0: + if len(member_types(cls)) == 0: if "configuration" in hints: found.append("has a configuration key, and the entity has no members") continue if "configuration" not in hints: found.append("has no configuration key, and the entity has members") continue - if ("configuration" in required) != compiled(cls).configuration_required: + if ("configuration" in required) != configuration_required(cls): found.append( "has configuration " + ("required" if "configuration" in required else "optional") + ", but a member is " - + ("required" if compiled(cls).configuration_required else "not required") + + ("required" if configuration_required(cls) else "not required") ) configuration = hints["configuration"] if not is_typeddict(configuration): @@ -679,21 +666,21 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | except NameError: continue keys = configuration_hints.keys() - if set(keys) != set(compiled(cls).member_types): + if set(keys) != set(member_types(cls)): found.append( f"has configuration keys {sorted(keys)!r} where the members are " - f"{sorted(compiled(cls).member_types)!r}" + f"{sorted(member_types(cls))!r}" ) continue - configuration_required = { + required_keys = { key for key, value in configuration_hints.items() if get_origin(value) is not NotRequired } misstated = sorted( key - for key, (member_required, _) in compiled(cls).member_types.items() - if (key in configuration_required) != member_required + for key, (member_required, _) in member_types(cls).items() + if (key in required_keys) != member_required ) if len(misstated) != 0: found.append( @@ -739,7 +726,7 @@ def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: default has to be unwrapped. """ defaulted: dict[str, object] = {} - for key in compiled(cls).member_types: + for key in member_types(cls): declared: object = getattr(cls, key, MISSING) if type(declared) is Field: spec = cast("Field[object]", declared) @@ -759,7 +746,7 @@ def _optional_members_default_to_unset(cls: type[MetadataEntity]) -> str | None: defaulted = _declared_defaults(cls) invented = [ key - for key, (required, _) in compiled(cls).member_types.items() + for key, (required, _) in member_types(cls).items() if not required and defaulted[key] is not UNSET ] if len(invented) == 0: @@ -780,7 +767,7 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: defaulted = _declared_defaults(cls) presumed = [ key - for key, (required, _) in compiled(cls).member_types.items() + for key, (required, _) in member_types(cls).items() if required and defaulted[key] is not MISSING ] if len(presumed) == 0: @@ -830,9 +817,8 @@ class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): and raises `MetadataValidationError` once -- so `BloscCodec(clevel=99)` raises, and `coerce` reports the same problems instead. `coerce`, `configuration`, `to_json` and `canonical` are written once here - against what the fields say; `compiled(cls)` is that reading, computed - once per class and kept beside it, so the class itself is never - written to. + against what the fields say, read off them by `member_types`, + `nested_members` and `name_members` as needed. """ extension_point: ClassVar[ExtensionPointField | None] = None @@ -875,8 +861,8 @@ class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: """Compile the entity from its fields, and refuse one this layer cannot use. - `compiled` reads the tables the layer needs off the fields -- the - members and their checks, the nested fields -- and then every + `member_types` is read off the fields first, which refuses an + annotation outside the shapes the compiler reads; then every invariant in `_INVARIANTS` is asked. Each names something that type-checks cleanly and then goes wrong later, somewhere that will not name this class; an import-time error in @@ -893,7 +879,7 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: # the first one's dict: verified already, and its members are # slot descriptors now rather than the defaults the checks read. return - compiled(cls) + member_types(cls) for invariant in _INVARIANTS: message = invariant(cls) if message is not None: @@ -919,22 +905,21 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: if name is None or not cls.accepts(name): return None, problem((), f"expected the {cls.identifier!r} entity") if configuration is None: - if compiled(cls).configuration_required: + if configuration_required(cls): return None, problem( ("configuration",), f"{cls.identifier!r} requires a configuration", "missing_key", ) configuration = cast("Mapping[str, object]", {}) - spec = compiled(cls) - members, own = coerce_members(configuration, spec.member_types) - for member in spec.name_members: + members, own = coerce_members(configuration, member_types(cls)) + for member in name_members(cls): members[member] = name # A member that is itself an entity is read in the scope whatever # else was found: its problems are determinable, so they are # reported in the same pass. found = own - for member, annotation in spec.nested_members.items(): + for member, annotation in nested_members(cls).items(): if member in members: members[member], nested = _resolve( annotation, members[member], context, ("configuration", member) @@ -979,7 +964,7 @@ def canonical(self) -> Self: the entity's own rewrite, which is where an entity says that two spellings of its own members mean the same. """ - nested = compiled(type(self)).nested_members + nested = nested_members(type(self)) walked = ( self if len(nested) == 0 @@ -1026,7 +1011,7 @@ def configuration(self) -> dict[str, object]: through the document it returned. """ members = self._configuration_members() - for name, annotation in compiled(type(self)).nested_members.items(): + for name, annotation in nested_members(type(self)).items(): if name in members: members[name] = render_nested(annotation, members[name]) return deepcopy(members) @@ -1039,7 +1024,7 @@ def _configuration_members(self) -> dict[str, object]: """ return { key: value - for key in compiled(type(self)).member_types + for key in member_types(type(self)) if (value := getattr(self, key)) is not UNSET } @@ -1219,7 +1204,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "CodecEntity", "CodecKind", "Coerced", - "Compiled", "DataTypeEntity", "ExtensionPointField", "JSONT_co", @@ -1230,7 +1214,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "StorageClass", "TypeCheck", "coerce_members", - "compiled", "is_bool", "is_entity", "is_int", From 48d0dfd969f03315d86fb0be0f49f42612cdf5ed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 15:30:03 +0200 Subject: [PATCH 080/107] refactor(zarr-metadata): coerce type-checks the configuration against the fields as it reads Review question: why is anything defined outside the individual classes? The entity declares its fields and one `__post_init__`; the base's `coerce` type-checks a document's configuration against those fields as it reads them, member by member, resolving the ones that hold entities through the scope, and builds the entity only when every member read. That needs no table: `member_types`, `nested_members`, `name_members`, `configuration_required`, `derive_member_types`, `coerce_members` and the `MemberTypes` alias are gone, along with the cache that held them. The invariants read the same field hints; the first of them refuses a field annotation outside the shapes the compiler reads, with the message that named the table's failure before. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 4 +- packages/zarr-metadata/changes/4379.misc.2.md | 10 +- .../src/zarr_metadata/v3/_checks.py | 59 +----- .../src/zarr_metadata/v3/_compile.py | 55 ++--- .../src/zarr_metadata/v3/_entity.py | 192 +++++++++--------- 5 files changed, 134 insertions(+), 186 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index ae106a9c68..403a7ffccc 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -38,8 +38,8 @@ metadata field -- an entity type, with or without `Opaque`. That covers every member this package models; an annotation outside them is refused at class creation, and `Annotated[str, FROM_NAME]` marks the one field carried by the envelope's name rather than a configuration key (`r`), -which `coerce` fills from the envelope. `configuration_required` follows -too, since the spec ties the bare-name spelling to whether any member is +which `coerce` fills from the envelope. Whether a configuration is required +follows too, since the spec ties the bare-name spelling to whether any member is required. The public JSON TypedDict an entity names as its base's argument is held to the fields at class creation, key by key. diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 25dc36dc0f..57423a6acb 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -45,8 +45,8 @@ a declared tuple of named functions, asks each invariant of the compiled class in order. `canonical` is `@final`, so an override is refused by pyright in the author's editor as well as at class creation; the guards that refused `problems` and `prepare` -- names from earlier drafts of this branch, never released -- are gone, -and declaring any derived class variable is refused, where before only -`configuration_required` was. +and nothing is kept on the class: what the layer needs of an entity's +fields, it reads off them when it reads a document. Four adversarial reviews -- two extension authors writing a codec and a data type against the door alone, a design review, an onboarding review @@ -74,9 +74,9 @@ must answer is abstract on it -- `transition` on `ArrayArrayCodec`, -- so a codec's kind is its base class (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`) rather than a string it sets, and an entity that leaves a hook undefined is refused at registration -rather than accepted with a silent default. The tables the layer reads -off an entity's fields live in `compiled(cls)`, a record kept beside the -class: a class is not written to after its definition. The compiler +rather than accepted with a silent default. Nothing is derived from an +entity's fields ahead of time: `coerce` type-checks a configuration +against them as it reads it. The compiler recognises a nested metadata field by a marker base, `MetadataFieldValue`, rather than by a module attribute set from outside. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py index 78a0bc6d06..a9762b624a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py @@ -1,11 +1,9 @@ -"""The member checks every entity needs, and the walk that applies them. +"""The member checks every entity needs. One check is a function of a value and its location that returns the problems it found -- none, for a value of the right type. The scalars, -a closed set of names, a homogeneous sequence, and a nested metadata -field cover what a configuration member can be; `coerce_members` walks -a configuration with a table of them, distinguishing an unknown key, an -optional member that failed, and a required one that did. `within` and +a closed set of names, a homogeneous sequence, an object, and a nested +metadata field cover what a configuration member can be. `within` and `named_configuration` are how an entity's problems and envelope are read from the document that holds it. """ @@ -13,7 +11,7 @@ from __future__ import annotations # Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below -# (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class +# (`TypeCheck`) are resolved by `get_type_hints` at class # creation, and a name that exists only for the type checker is a NameError # then -- for this package and for any tool introspecting an entity. from collections.abc import Callable, Mapping, Sequence @@ -41,10 +39,6 @@ """Whether one value has the type a member declares, and where if not.""" -MemberTypes: TypeAlias = "Mapping[str, tuple[bool, TypeCheck]]" -"""Per configuration member: whether it is required, and its type check.""" - - def problem( loc: Loc, message: str, kind: ProblemKind = "invalid_type" ) -> tuple[ValidationProblem, ...]: @@ -141,7 +135,7 @@ def check(candidate: object, loc: Loc) -> tuple[ValidationProblem, ...]: return check -def _as_tuples(value: object) -> object: +def as_tuples(value: object) -> object: """Every JSON array in `value`, at any depth, as a tuple. The TypedDicts spell a JSON array as a tuple throughout, so a member @@ -151,49 +145,13 @@ def _as_tuples(value: object) -> object: """ if isinstance(value, (list, tuple)): entries = cast("list[object] | tuple[object, ...]", value) - return tuple(_as_tuples(entry) for entry in entries) + return tuple(as_tuples(entry) for entry in entries) if isinstance(value, Mapping): entries = cast("Mapping[str, object]", value) - return {key: _as_tuples(entry) for key, entry in entries.items()} + return {key: as_tuples(entry) for key, entry in entries.items()} return value -def coerce_members( - configuration: Mapping[str, object], types: MemberTypes -) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: - """The members `types` declares, taken from `configuration`. - - Returns what was read and every problem found. A key the entity does - not declare says the value carries something extra, not that it is - wrong, so the member it sits beside is still read; a member of the - wrong type, or a required one missing, is reported and left out -- - and an entity is never built around the hole, because its rules are - written over a whole configuration. - """ - problems: list[ValidationProblem] = [] - members: dict[str, object] = {} - for key in configuration: - if key not in types: - problems.extend( - problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") - ) - for key, (required, check) in types.items(): - if key not in configuration: - if required: - problems.extend( - problem(("configuration", key), f"missing required key {key!r}", "missing_key") - ) - continue - # Normalized before the check, so a check only ever sees the tuples - # the TypedDicts declare -- never the lists raw JSON arrives as. - value = _as_tuples(configuration[key]) - found = check(value, ("configuration", key)) - problems.extend(found) - if all(entry.kind == "unknown_key" for entry in found): - members[key] = value - return members, tuple(problems) - - def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: """A nested metadata field: a bare name or a named-configuration object. @@ -252,9 +210,8 @@ def named_configuration( __all__ = [ "Loc", - "MemberTypes", "TypeCheck", - "coerce_members", + "as_tuples", "is_bool", "is_int", "is_integer", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py index b33e344829..f21528f63e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py @@ -5,8 +5,7 @@ -- a scalar, a `Literal`, arrays homogeneous or fixed, unions, a nested object described by a TypedDict or a record dataclass, an object of undeclared keys, a `NewType` as the type it names: the shapes JSON takes -and no others, which is what keeps it small. `derive_member_types` is -what an entity reads its member table off. Anything finer than a type +and no others, which is what keeps it small. Anything finer than a type -- a bound, a rule about a member, members read together -- is the entity's own `__post_init__`, in plain code. @@ -18,7 +17,7 @@ from __future__ import annotations # Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below -# (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class +# (`TypeCheck`) are resolved by `get_type_hints` at class # creation, and a name that exists only for the type checker is a NameError # then -- for this package and for any tool introspecting an entity. import sys @@ -416,6 +415,20 @@ def _compile_new_type(inner: object) -> TypeCheck | None: return check_for(cast("NewType", inner).__supertype__) +def is_from_name(annotation: object) -> bool: + """Whether `FROM_NAME` marks the field: carried by the envelope's name, not a configuration key.""" + return any(entry is FROM_NAME for entry in strip_annotation(annotation)[1]) + + +def type_check(annotation: object) -> TypeCheck: + """The type check a field annotation implies; `TypeError` if it implies none.""" + check = check_for(annotation) + if check is None: + msg = f"{annotation!r} is not a shape JSON takes" + raise TypeError(msg) + return check + + def check_for(annotation: object) -> TypeCheck | None: """The type check a field annotation implies, or None if it implies none. @@ -464,38 +477,6 @@ def check_for(annotation: object) -> TypeCheck | None: return None -def envelope_members(cls: type) -> tuple[str, ...]: - """The fields `FROM_NAME` marks: carried by the envelope's name, not by a configuration key.""" - return tuple( - name - for name, annotation in field_hints(cls).items() - if any(entry is FROM_NAME for entry in strip_annotation(annotation)[1]) - ) - - -def derive_member_types(cls: type) -> tuple[dict[str, tuple[bool, TypeCheck]], list[str]]: - """The member table an entity's own fields describe. - - Every field is a configuration member unless `FROM_NAME` says it is - carried by the envelope. Requiredness is whether the type admits - `UNSET`; the check is whatever `check_for` reads off the type. Also - returned: the fields no check could be read for, which class - creation refuses. - """ - derived: dict[str, tuple[bool, TypeCheck]] = {} - unread: list[str] = [] - for name, annotation in field_hints(cls).items(): - inner, metadata = strip_annotation(annotation) - if any(entry is FROM_NAME for entry in metadata): - continue - check = check_for(inner) - if check is None: - unread.append(name) - continue - derived[name] = (not is_optional(inner), check) - return derived, unread - - def element_annotations(inner: object, count: int) -> list[object]: """The annotation of each element of a tuple type, one per element held.""" arguments = get_args(inner) @@ -539,14 +520,13 @@ def declared_class_vars(cls: type) -> dict[str, type]: "any_of", "check_for", "declared_class_vars", - "derive_member_types", "describe", "element_annotations", - "envelope_members", "field_hints", "fixed_tuple", "has_shape", "is_class_var", + "is_from_name", "is_metadata_field_type", "is_nested_field", "is_optional", @@ -555,5 +535,6 @@ def declared_class_vars(cls: type) -> dict[str, type]: "own_annotations", "shape_of", "strip_annotation", + "type_check", "unsubscripted", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index fc9dc740cc..c167b598be 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -36,8 +36,6 @@ from collections.abc import Callable, Mapping # noqa: TC003 from copy import deepcopy from dataclasses import MISSING, Field, dataclass, is_dataclass, replace -from functools import cache -from types import MappingProxyType from typing import ( TYPE_CHECKING, ClassVar, @@ -69,9 +67,8 @@ from zarr_metadata.v3._checks import ( Loc, - MemberTypes, TypeCheck, - coerce_members, + as_tuples, is_bool, is_int, is_integer, @@ -88,19 +85,21 @@ from zarr_metadata.v3._compile import ( FROM_NAME, MetadataFieldValue, + check_for, declared_class_vars, - derive_member_types, element_annotations, - envelope_members, field_hints, has_shape, is_class_var, + is_from_name, is_metadata_field_type, is_nested_field, + is_optional, is_union, own_annotations, shape_of, strip_annotation, + type_check, unsubscripted, ) @@ -428,50 +427,42 @@ class Opaque(MetadataFieldValue): """What to do instead, for each method the base marks `@final`.""" -@cache -def member_types(cls: type[MetadataEntity]) -> MemberTypes: - """The configuration members, and the type check each one's annotation implies. +def _members(cls: type[MetadataEntity]) -> dict[str, bool]: + """The configuration members, each with whether it is required: the fields, less the envelope's.""" + return { + name: not is_optional(annotation) + for name, annotation in field_hints(cls).items() + if not is_from_name(annotation) + } - Read off the fields. Raises `TypeError` for a field annotation outside - the shapes the compiler reads. - """ - derived, unread = derive_member_types(cls) - if len(unread) != 0: - hints = field_hints(cls) - msg = ( - f"{cls.__name__}: " - f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in sorted(unread))}" - ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " - "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " - "record, Mapping[str, V], a NewType, or an entity kind with Opaque " - "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in `__post_init__`" - ) - raise TypeError(msg) - return MappingProxyType(derived) - - -def configuration_required(cls: type[MetadataEntity]) -> bool: - """Whether some member is required, which is what makes the bare-name spelling illegal.""" - return any(required for required, _ in member_types(cls).values()) - -@cache -def nested_members(cls: type[MetadataEntity]) -> Mapping[str, object]: +def _nested(cls: type[MetadataEntity]) -> dict[str, object]: """The fields that hold other entities, with their annotations.""" - return MappingProxyType( - { - name: annotation - for name, annotation in field_hints(cls).items() - if contains_entity(annotation) - } - ) + return { + name: annotation + for name, annotation in field_hints(cls).items() + if contains_entity(annotation) + } -@cache -def name_members(cls: type[MetadataEntity]) -> tuple[str, ...]: - """The fields the envelope's name carries, marked `Annotated[str, FROM_NAME]`.""" - return envelope_members(cls) +def _fields_are_json_shapes(cls: type[MetadataEntity]) -> str | None: + hints = field_hints(cls) + unread = sorted( + name + for name, annotation in hints.items() + if not is_from_name(annotation) and check_for(annotation) is None + ) + if len(unread) == 0: + return None + return ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in unread)}" + ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in `__post_init__`" + ) def _final_methods_are_not_overridden(cls: type[MetadataEntity]) -> str | None: @@ -488,7 +479,7 @@ def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: # field typed as one could not be resolved through a scope. unplaced = sorted( name - for name, annotation in nested_members(cls).items() + for name, annotation in _nested(cls).items() if any(kind.extension_point is None for kind in _entity_kinds(annotation)) ) if len(unplaced) == 0: @@ -525,7 +516,7 @@ def _nested_fields_admit_opaque(cls: type[MetadataEntity]) -> str | None: # `codec.inner.level` would be accepted and then raise. lacking = sorted( name - for name, annotation in nested_members(cls).items() + for name, annotation in _nested(cls).items() if _entity_unions_lacking_opaque(annotation) ) if len(lacking) == 0: @@ -538,8 +529,8 @@ def _nested_fields_admit_opaque(cls: type[MetadataEntity]) -> str | None: def _fields_do_not_shadow_class_variables(cls: type[MetadataEntity]) -> str | None: - # A field of that name would go into `member_types`, into the - # configuration, and into the JSON -- while the class variable it + # A field of that name would go into the configuration and into the + # JSON -- while the class variable it # shadows is what every other part of this layer reads. annotated = declared_class_vars(cls) shadowed = [ @@ -605,8 +596,9 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | f"{cls.__name__} names {json_type!r} as its JSON type, which is not an object " "TypedDict, a name type, or a union of one of each" ) - writes_bare = not configuration_required(cls) and cls.must_understand - writes_object = len(member_types(cls)) != 0 or not cls.must_understand + members = _members(cls) + writes_bare = not any(members.values()) and cls.must_understand + writes_object = len(members) != 0 or not cls.must_understand found: list[str] = [] if writes_bare and len(names) == 0: found.append("lacks the bare name the entity writes when every member is absent") @@ -644,19 +636,19 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | ) if not cls.must_understand and "must_understand" not in hints: found.append("has no must_understand key, which the entity writes") - if len(member_types(cls)) == 0: + if len(members) == 0: if "configuration" in hints: found.append("has a configuration key, and the entity has no members") continue if "configuration" not in hints: found.append("has no configuration key, and the entity has members") continue - if ("configuration" in required) != configuration_required(cls): + if ("configuration" in required) != any(members.values()): found.append( "has configuration " + ("required" if "configuration" in required else "optional") + ", but a member is " - + ("required" if configuration_required(cls) else "not required") + + ("required" if any(members.values()) else "not required") ) configuration = hints["configuration"] if not is_typeddict(configuration): @@ -666,10 +658,9 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | except NameError: continue keys = configuration_hints.keys() - if set(keys) != set(member_types(cls)): + if set(keys) != set(members): found.append( - f"has configuration keys {sorted(keys)!r} where the members are " - f"{sorted(member_types(cls))!r}" + f"has configuration keys {sorted(keys)!r} where the members are {sorted(members)!r}" ) continue required_keys = { @@ -679,7 +670,7 @@ def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | } misstated = sorted( key - for key, (member_required, _) in member_types(cls).items() + for key, member_required in members.items() if (key in required_keys) != member_required ) if len(misstated) != 0: @@ -726,7 +717,7 @@ def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: default has to be unwrapped. """ defaulted: dict[str, object] = {} - for key in member_types(cls): + for key in _members(cls): declared: object = getattr(cls, key, MISSING) if type(declared) is Field: spec = cast("Field[object]", declared) @@ -746,7 +737,7 @@ def _optional_members_default_to_unset(cls: type[MetadataEntity]) -> str | None: defaulted = _declared_defaults(cls) invented = [ key - for key, (required, _) in member_types(cls).items() + for key, required in _members(cls).items() if not required and defaulted[key] is not UNSET ] if len(invented) == 0: @@ -766,9 +757,7 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: # named so that asking for one is deliberate. defaulted = _declared_defaults(cls) presumed = [ - key - for key, (required, _) in member_types(cls).items() - if required and defaulted[key] is not MISSING + key for key, required in _members(cls).items() if required and defaulted[key] is not MISSING ] if len(presumed) == 0: return None @@ -779,6 +768,7 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: _INVARIANTS: Final[tuple[Callable[[type[MetadataEntity]], str | None], ...]] = ( + _fields_are_json_shapes, _final_methods_are_not_overridden, _nested_kinds_have_a_point, _nested_fields_admit_opaque, @@ -817,8 +807,7 @@ class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): and raises `MetadataValidationError` once -- so `BloscCodec(clevel=99)` raises, and `coerce` reports the same problems instead. `coerce`, `configuration`, `to_json` and `canonical` are written once here - against what the fields say, read off them by `member_types`, - `nested_members` and `name_members` as needed. + against what the fields say, read off them as needed. """ extension_point: ClassVar[ExtensionPointField | None] = None @@ -861,9 +850,8 @@ class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: """Compile the entity from its fields, and refuse one this layer cannot use. - `member_types` is read off the fields first, which refuses an - annotation outside the shapes the compiler reads; then every - invariant in `_INVARIANTS` is asked. Each names + Every invariant in `_INVARIANTS` is asked, the first of which + refuses a field annotation outside the shapes the compiler reads. Each names something that type-checks cleanly and then goes wrong later, somewhere that will not name this class; an import-time error in the extension's own module is the one place the author is looking. @@ -879,7 +867,6 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: # the first one's dict: verified already, and its members are # slot descriptors now rather than the defaults the checks read. return - member_types(cls) for invariant in _INVARIANTS: message = invariant(cls) if message is not None: @@ -904,48 +891,75 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: name, configuration, _ = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), f"expected the {cls.identifier!r} entity") + hints = field_hints(cls) if configuration is None: - if configuration_required(cls): + if any(_members(cls).values()): return None, problem( ("configuration",), f"{cls.identifier!r} requires a configuration", "missing_key", ) configuration = cast("Mapping[str, object]", {}) - members, own = coerce_members(configuration, member_types(cls)) - for member in name_members(cls): - members[member] = name + members: dict[str, object] = {} + found: list[ValidationProblem] = [] + for key in configuration: + if key not in hints or is_from_name(hints[key]): + found.extend( + problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") + ) + for key, annotation in hints.items(): + if is_from_name(annotation): + members[key] = name + continue + if key not in configuration: + if not is_optional(annotation): + found.extend( + problem( + ("configuration", key), f"missing required key {key!r}", "missing_key" + ) + ) + continue + # Normalized before the check, so a check only ever sees the + # tuples the TypedDicts declare, never the lists raw JSON + # arrives as. A member of the wrong type is reported and left + # out; an unknown key inside it is survivable. + member = as_tuples(configuration[key]) + problems = type_check(annotation)(member, ("configuration", key)) + found.extend(problems) + if all(entry.kind == "unknown_key" for entry in problems): + members[key] = member + own = tuple(found) # A member that is itself an entity is read in the scope whatever # else was found: its problems are determinable, so they are # reported in the same pass. - found = own - for member, annotation in nested_members(cls).items(): - if member in members: - members[member], nested = _resolve( - annotation, members[member], context, ("configuration", member) + for key, annotation in hints.items(): + if key in members and contains_entity(annotation): + members[key], nested = _resolve( + annotation, members[key], context, ("configuration", key) ) - found = (*found, *nested) + found.extend(nested) + found_all = tuple(found) if any(entry.kind != "unknown_key" for entry in own): # One of this entity's own members could not be read. That # leaves a hole, and the rules are written over a whole # configuration -- blosc's `typesize` requirement reads # `shuffle` -- so judging around it would be guessing: the # entity is not built, and the type problems stand alone. - return None, found + return None, found_all try: entity = cls(**members) except MetadataValidationError as refused: # `__post_init__` found values the spec disallows: reported # rather than raised, located under the configuration. - return None, (*found, *within((), refused.problems)) - if any(entry.kind != "unknown_key" for entry in found): + return None, (*found_all, *within((), refused.problems)) + if any(entry.kind != "unknown_key" for entry in found_all): # A contained entity could not be read. This entity's own # rules ran -- an invalid inner is an `Opaque`, as an # out-of-scope one is -- but what is handed back is not an # entity that would be asked composition questions it cannot # answer. - return None, found - return entity, found + return None, found_all + return entity, found_all @final def canonical(self) -> Self: @@ -964,7 +978,7 @@ def canonical(self) -> Self: the entity's own rewrite, which is where an entity says that two spellings of its own members mean the same. """ - nested = nested_members(type(self)) + nested = _nested(type(self)) walked = ( self if len(nested) == 0 @@ -1011,7 +1025,7 @@ def configuration(self) -> dict[str, object]: through the document it returned. """ members = self._configuration_members() - for name, annotation in nested_members(type(self)).items(): + for name, annotation in _nested(type(self)).items(): if name in members: members[name] = render_nested(annotation, members[name]) return deepcopy(members) @@ -1023,9 +1037,7 @@ def _configuration_members(self) -> dict[str, object]: `r` width, which lives in the name. """ return { - key: value - for key in member_types(type(self)) - if (value := getattr(self, key)) is not UNSET + key: value for key in _members(type(self)) if (value := getattr(self, key)) is not UNSET } def to_json(self) -> JSONT_co: @@ -1208,12 +1220,10 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "ExtensionPointField", "JSONT_co", "Loc", - "MemberTypes", "MetadataEntity", "Opaque", "StorageClass", "TypeCheck", - "coerce_members", "is_bool", "is_entity", "is_int", From e6e3f946949af2c57331b0139b3015433faacd84 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 15:40:38 +0200 Subject: [PATCH 081/107] refactor(zarr-metadata): a scope holds entities by kind; the document names its fields Review question: why is `ExtensionPointField` defined outside the array document's validation logic? It was the array document's field names, carried into the entity layer as the key of every registry table, the `extension_point` class variable on every kind, and five constants. That knowledge belongs to the document. Now a scope holds entities by kind -- `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity`, `CodecEntity`, `StorageTransformerEntity`, the last two new, so every entity is of one -- and `read_array_v3` is the one place that says which field holds which kind. `Context.of(*classes)` and `extended_with(*classes)` read each class's kind off its base and its key off its `identifier`, so nothing can be misfiled and the two registration checks for that are gone; `resolve` and `coerce` take the kind (or any subclass, so a field typed `GzipCodec | Opaque` resolves only gzip) and are generic in it, which retires four overloads apiece and the casts that funnelled the document's fields through one dict. The typed-per-point TypedDicts, `_ENTITY_KINDS` and `_point_of` go with them. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 8 + .../src/zarr_metadata/v3/_document.py | 101 ++-- .../src/zarr_metadata/v3/_entity.py | 127 ++--- .../src/zarr_metadata/v3/_registry.py | 462 ++++++------------ .../v3/chunk_key_encoding/default.py | 4 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 4 +- .../src/zarr_metadata/v3/entity.py | 44 +- .../tests/rules/test_chain_properties.py | 4 +- .../tests/rules/test_chunk_grid.py | 5 +- .../zarr-metadata/tests/test_public_api.py | 2 - .../tests/v3/test_acme_affine.py | 2 +- .../tests/v3/test_acme_decimal.py | 4 +- .../zarr-metadata/tests/v3/test_entities.py | 75 +-- .../tests/v3/test_extension_api.py | 81 ++- .../tests/v3/test_fill_values.py | 6 +- .../zarr-metadata/tests/v3/test_resolve.py | 43 +- 16 files changed, 381 insertions(+), 591 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 57423a6acb..356d8d2dd5 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -80,6 +80,14 @@ against them as it reads it. The compiler recognises a nested metadata field by a marker base, `MetadataFieldValue`, rather than by a module attribute set from outside. +A scope holds entities by kind -- `DataTypeEntity`, `ChunkGridEntity`, +`ChunkKeyEncodingEntity`, `CodecEntity`, `StorageTransformerEntity` -- +and `extended_with` takes classes, reading each one's kind off its base +and its key off its `identifier`, so nothing can be misfiled and the two +registration checks for that are gone. Which of a document's fields +holds which kind is the document's own knowledge, in one place; the +entity layer no longer names a document field. + One thing this does not change, under mypy. An entity's JSON type is a TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as `Mapping[str, object]`, never as the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 17e5bdfcf2..0f1ed2d6a6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -18,7 +18,7 @@ from collections.abc import Mapping from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Final, cast +from typing import TYPE_CHECKING, TypeVar, cast from zarr_metadata.model._validation import ( MetadataValidationError, @@ -31,17 +31,13 @@ from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._compile import field_hints from zarr_metadata.v3._entity import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, - STORAGE_TRANSFORMERS, ChunkGridEntity, + ChunkKeyEncodingEntity, CodecEntity, DataTypeEntity, - ExtensionPointField, MetadataEntity, Opaque, + StorageTransformerEntity, canonicalize_nested, contains_entity, render_nested, @@ -56,6 +52,9 @@ from zarr_metadata.v3._entity import Loc +_EntityT = TypeVar("_EntityT", bound=MetadataEntity) + + @dataclass(frozen=True, slots=True) class ArrayDocumentV3: """A v3 array document with its extension points read as entities. @@ -70,9 +69,9 @@ class ArrayDocumentV3: document: Mapping[str, object] data_type: DataTypeEntity | Opaque chunk_grid: ChunkGridEntity | Opaque - chunk_key_encoding: MetadataEntity | Opaque + chunk_key_encoding: ChunkKeyEncodingEntity | Opaque codecs: tuple[CodecEntity | Opaque, ...] - storage_transformers: tuple[MetadataEntity | Opaque, ...] + storage_transformers: tuple[StorageTransformerEntity | Opaque, ...] def problems(self) -> tuple[ValidationProblem, ...]: """Every semantic problem this document has, once it has been read. @@ -173,21 +172,30 @@ def parts(self) -> ArrayParts: ) -# The three extension points a document names once, and the field each is -# named in. `codecs` is the fourth and holds a list, so it is separate. -_SINGLE_FIELDS: Final[tuple[tuple[ExtensionPointField, str], ...]] = ( - (DATA_TYPE, "data_type"), - (CHUNK_GRID, "chunk_grid"), - (CHUNK_KEY_ENCODING, "chunk_key_encoding"), -) - -# The two the document names as a list. Nothing models a storage -# transformer yet, so nothing is judged there today -- but the extension -# point is registerable, and a registered one has to be reached. -_SEQUENCE_FIELDS: Final[tuple[tuple[ExtensionPointField, str], ...]] = ( - (CODECS, "codecs"), - (STORAGE_TRANSFORMERS, "storage_transformers"), -) +def _read_one( + context: Context, kind: type[_EntityT], document: Mapping[str, object], key: str +) -> tuple[_EntityT | Opaque, tuple[ValidationProblem, ...]]: + """The entity of `kind` the document names at `key`; an `Opaque` if it names none.""" + value = document.get(key) + if value is None: + return Opaque(None, "invalid"), () + return context.coerce(kind, value, (key,), envelope_judged=True) + + +def _read_each( + context: Context, kind: type[_EntityT], document: Mapping[str, object], key: str +) -> tuple[tuple[_EntityT | Opaque, ...], tuple[ValidationProblem, ...]]: + """The entities of `kind` the document lists at `key`, in order.""" + entries = document.get(key) + if not isinstance(entries, (list, tuple)): + return (), () + read: list[_EntityT | Opaque] = [] + problems: list[ValidationProblem] = [] + for index, entry in enumerate(cast("Sequence[object]", entries)): + entity, found = context.coerce(kind, entry, (key, index), envelope_judged=True) + read.append(entity) + problems.extend(found) + return tuple(read), tuple(problems) def read_array_v3( @@ -195,39 +203,28 @@ def read_array_v3( ) -> tuple[ArrayDocumentV3, tuple[ValidationProblem, ...]]: """`document`'s extension points, read in `context`. - Type-space only: what comes back is well-typed by construction, and - the problems are the reasons some of it is not an entity. + The one place that knows which of a document's fields holds which + kind of entity. Type-space only: what comes back is well-typed by + construction, and the problems are the reasons some of it is not an + entity. """ - read: dict[str, MetadataEntity | Opaque] = {} - problems: list[ValidationProblem] = [] - for field, key in _SINGLE_FIELDS: - value = document.get(key) - if value is None: - read[key] = Opaque(None, "invalid") - continue - entity, found = context.coerce(field, value, (key,), envelope_judged=True) - read[key] = entity - problems.extend(found) - sequences: dict[str, tuple[MetadataEntity | Opaque, ...]] = {} - for field, key in _SEQUENCE_FIELDS: - read_entries: list[MetadataEntity | Opaque] = [] - entries = document.get(key) - if isinstance(entries, (list, tuple)): - for index, entry in enumerate(cast("Sequence[object]", entries)): - entity, found = context.coerce(field, entry, (key, index), envelope_judged=True) - read_entries.append(entity) - problems.extend(found) - sequences[key] = tuple(read_entries) + data_type, found_1 = _read_one(context, DataTypeEntity, document, "data_type") + chunk_grid, found_2 = _read_one(context, ChunkGridEntity, document, "chunk_grid") + encoding, found_3 = _read_one(context, ChunkKeyEncodingEntity, document, "chunk_key_encoding") + codecs, found_4 = _read_each(context, CodecEntity, document, "codecs") + transformers, found_5 = _read_each( + context, StorageTransformerEntity, document, "storage_transformers" + ) return ( ArrayDocumentV3( document=document, - data_type=cast("DataTypeEntity | Opaque", read["data_type"]), - chunk_grid=cast("ChunkGridEntity | Opaque", read["chunk_grid"]), - chunk_key_encoding=read["chunk_key_encoding"], - codecs=cast("tuple[CodecEntity | Opaque, ...]", sequences["codecs"]), - storage_transformers=sequences["storage_transformers"], + data_type=data_type, + chunk_grid=chunk_grid, + chunk_key_encoding=encoding, + codecs=codecs, + storage_transformers=transformers, ), - tuple(problems), + (*found_1, *found_2, *found_3, *found_4, *found_5), ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index c167b598be..6f1d753814 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -139,37 +139,7 @@ class GzipCodec(CodecEntity[GzipCodecMetadata]): ... """ -ExtensionPointField = Literal[ - "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" -] -"""The v3 array metadata fields whose values name an extension. - -Names are unique only within a point -- `bytes` is both a core codec and -a registered data type -- so every table in this package is keyed by -point and then by name, and an entity that contains other entities says -which point it reads them at. -""" - - # Left to infer their `Literal` types rather than widened to -# `ExtensionPointField`: `Context.coerce` overloads on the field, so a -# call written with one of these constants gets the entity type back -# rather than the base. They are still assignable to the alias. -DATA_TYPE: Final = "data_type" - - -CHUNK_GRID: Final = "chunk_grid" - - -CHUNK_KEY_ENCODING: Final = "chunk_key_encoding" - - -CODECS: Final = "codecs" - - -STORAGE_TRANSFORMERS: Final = "storage_transformers" - - StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How one scalar of a data type occupies bytes. @@ -270,20 +240,6 @@ def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: return [] -def _point_of(kind: type[MetadataEntity]) -> ExtensionPointField: - """The point a nested entity kind is resolved at. - - Every nested field's kind has one by the time an entity exists -- - `__init_subclass__` refuses the class otherwise -- so this is the - narrowing, not a second check. - """ - point = kind.extension_point - if point is None: - msg = f"{kind.__name__} is registered at no single extension point" - raise TypeError(msg) - return point - - def _fitting_branch(inner: object, value: object) -> object | None: """The branch of a union that holds an entity and whose shape `value` has.""" for branch in get_args(inner): @@ -306,7 +262,7 @@ def _resolve( """ inner, _ = strip_annotation(annotation) if is_nested_field(inner): - return context.coerce(_point_of(_entity_kinds(inner)[0]), value, loc) + return context.coerce(_entity_kinds(inner)[0], value, loc) if is_union(inner): branch = _fitting_branch(inner, value) return (value, ()) if branch is None else _resolve(branch, value, context, loc) @@ -474,20 +430,32 @@ def _final_methods_are_not_overridden(cls: type[MetadataEntity]) -> str | None: return None -def _nested_kinds_have_a_point(cls: type[MetadataEntity]) -> str | None: - # `MetadataEntity` itself is registered at no single point, so a - # field typed as one could not be resolved through a scope. +def _entities_are_of_a_kind(cls: type[MetadataEntity]) -> str | None: + # A scope holds entities by kind, so an entity of none could never be + # registered or resolved. + if kind_of(cls) is not None: + return None + return ( + f"{cls.__name__} subclasses MetadataEntity directly; subclass the kind of thing it is: " + "a codec kind, DataTypeEntity, ChunkGridEntity, ChunkKeyEncodingEntity or " + "StorageTransformerEntity" + ) + + +def _nested_fields_name_a_kind(cls: type[MetadataEntity]) -> str | None: + # A field typed as bare `MetadataEntity` could not be resolved through + # a scope: nothing says which kind's table to look in. unplaced = sorted( name for name, annotation in _nested(cls).items() - if any(kind.extension_point is None for kind in _entity_kinds(annotation)) + if any(kind_of(kind) is None for kind in _entity_kinds(annotation)) ) if len(unplaced) == 0: return None return ( - f"{cls.__name__}: the entity kind of {', '.join(unplaced)} has no " - "`extension_point`; annotate it with `CodecEntity`, `DataTypeEntity` " - "or `ChunkGridEntity`" + f"{cls.__name__}: the entity type of {', '.join(unplaced)} is of no kind; annotate it " + "with a codec kind, DataTypeEntity, ChunkGridEntity, ChunkKeyEncodingEntity or " + "StorageTransformerEntity, or a subclass of one" ) @@ -770,7 +738,8 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: _INVARIANTS: Final[tuple[Callable[[type[MetadataEntity]], str | None], ...]] = ( _fields_are_json_shapes, _final_methods_are_not_overridden, - _nested_kinds_have_a_point, + _entities_are_of_a_kind, + _nested_fields_name_a_kind, _nested_fields_admit_opaque, _fields_do_not_shadow_class_variables, _owed_class_variables_are_declared, @@ -810,16 +779,6 @@ class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): against what the fields say, read off them as needed. """ - extension_point: ClassVar[ExtensionPointField | None] = None - """Where this kind of entity is registered, if it is registered at one point. - - Set by `CodecEntity`, `DataTypeEntity` and `ChunkGridEntity`. It is - what makes a field typed as one of those resolvable: the scope is - asked at that point. `MetadataEntity` itself is the kind of the two - points that take any entity, so it names none, and a field typed as - bare `MetadataEntity` is refused at class creation. - """ - must_understand: ClassVar[bool] = True """Whether a reader must understand this entity to read the array. @@ -1092,9 +1051,6 @@ class CodecEntity(MetadataEntity[JSONT_co], base=True): pipeline the codec may stand, and what it must answer. """ - extension_point: ClassVar[ExtensionPointField] = CODECS - """Where a codec is registered, and so where a field typed as one is resolved.""" - kind: ClassVar[CodecKind] """Set by the kind class.""" @@ -1151,8 +1107,6 @@ class BytesBytesCodec(CodecEntity[JSONT_co], base=True): class ChunkGridEntity(MetadataEntity[JSONT_co], base=True): """An entity that divides an array into the parts a pipeline encodes.""" - extension_point: ClassVar[ExtensionPointField] = CHUNK_GRID - def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """Why this grid does not divide an array of `array_shape`. @@ -1181,8 +1135,6 @@ class DataTypeEntity(MetadataEntity[JSONT_co], base=True): a table of names. """ - extension_point: ClassVar[ExtensionPointField] = DATA_TYPE - scalar_storage: ClassVar[StorageClass] def storage_class(self) -> StorageClass | None: @@ -1202,27 +1154,49 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP """ +@dataclass(frozen=True) +class ChunkKeyEncodingEntity(MetadataEntity[JSONT_co], base=True): + """An entity that says how a chunk's coordinates become a store key.""" + + +@dataclass(frozen=True) +class StorageTransformerEntity(MetadataEntity[JSONT_co], base=True): + """An entity that stands between the codec pipeline and the store.""" + + +KINDS: Final[tuple[type[MetadataEntity], ...]] = ( + DataTypeEntity, + ChunkGridEntity, + ChunkKeyEncodingEntity, + CodecEntity, + StorageTransformerEntity, +) +"""The kinds of entity: what a scope holds a table of, and what a document's fields are read as.""" + + +def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: + """The kind `cls` is of, or None for a class under none of them.""" + return next((kind for kind in cls.__mro__ if kind in KINDS), None) + + __all__ = [ - "CHUNK_GRID", - "CHUNK_KEY_ENCODING", - "CODECS", - "DATA_TYPE", "FROM_NAME", - "STORAGE_TRANSFORMERS", + "KINDS", "ArrayArrayCodec", "ArrayBytesCodec", "BytesBytesCodec", "ChunkGridEntity", + "ChunkKeyEncodingEntity", "CodecEntity", "CodecKind", "Coerced", "DataTypeEntity", - "ExtensionPointField", "JSONT_co", "Loc", "MetadataEntity", "Opaque", "StorageClass", + "StorageTransformerEntity", "TypeCheck", "is_bool", "is_entity", @@ -1232,6 +1206,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "is_metadata_field", "is_str", "json_type_of", + "kind_of", "named_configuration", "one_of", "problem", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 80b73c9b5b..8b1c3e2fe8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -1,9 +1,10 @@ """Which entities are in scope when raw JSON is coerced. The only registry the package needs. A validator reading a document has -to decide, for each extension point, which identifier maps to which -entity — and that decision is the *scope* it validates against, not a -property of the entities themselves. +to decide, for each kind of entity, which identifier maps to which class +— and that decision is the *scope* it validates against, not a property +of the entities themselves. Which of a document's fields holds which +kind is the document's own knowledge, in `_document`. Two scopes, because the question "is this document valid?" has two useful answers. `CORE` is what the Zarr v3 specification itself defines, so a @@ -22,10 +23,10 @@ from __future__ import annotations import inspect +from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Final, Literal, overload - -from typing_extensions import TypedDict, Unpack +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeVar from zarr_metadata.model._validation import ( ValidationProblem, @@ -33,16 +34,10 @@ ) from zarr_metadata.v3._compile import is_class_var, own_annotations from zarr_metadata.v3._entity import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, - STORAGE_TRANSFORMERS, - ChunkGridEntity, - CodecEntity, - DataTypeEntity, + KINDS, MetadataEntity, Opaque, + kind_of, named_configuration, ) from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid @@ -80,266 +75,99 @@ from zarr_metadata.v3.data_type.uint64 import Uint64DataType if TYPE_CHECKING: - from collections.abc import Mapping - - from zarr_metadata.v3._entity import ExtensionPointField, Loc - - -class EntityTables(TypedDict): - """Which entity is registered under which name, at each extension point. + from zarr_metadata.v3._entity import Loc - Typed per point rather than as one mapping, because an entity's kind - is a fact about where it may be registered: a codec at `data_type` - would satisfy a `Mapping[str, type[MetadataEntity]]` and then fail - the moment anything asked it for a storage class. Spelling the - correspondence here is what makes `resolve`'s per-point return type - true rather than asserted, and what turns a misfiling into an error - where the table is written. - """ - data_type: Mapping[str, type[DataTypeEntity]] - codecs: Mapping[str, type[CodecEntity]] - chunk_grid: Mapping[str, type[ChunkGridEntity]] - chunk_key_encoding: Mapping[str, type[MetadataEntity]] - storage_transformers: Mapping[str, type[MetadataEntity]] +if TYPE_CHECKING: + from zarr_metadata.v3._entity import Loc +_EntityT = TypeVar("_EntityT", bound=MetadataEntity) -class PartialEntityTables(TypedDict, total=False): - """`EntityTables` with every point optional: what `extended_with` takes. - - A reader registering a codec of its own says so and nothing else; - the points it does not name keep whatever the scope it extended had. - """ - - data_type: Mapping[str, type[DataTypeEntity]] - codecs: Mapping[str, type[CodecEntity]] - chunk_grid: Mapping[str, type[ChunkGridEntity]] - chunk_key_encoding: Mapping[str, type[MetadataEntity]] - storage_transformers: Mapping[str, type[MetadataEntity]] - - -_ENTITY_KINDS: Final[Mapping[ExtensionPointField, type[MetadataEntity]]] = { - DATA_TYPE: DataTypeEntity, - CODECS: CodecEntity, - CHUNK_GRID: ChunkGridEntity, - CHUNK_KEY_ENCODING: MetadataEntity, - STORAGE_TRANSFORMERS: MetadataEntity, -} -"""The base every entity at a point must derive from. - -`EntityTables` says the same thing to the type checker, which is where a -table written out in source is caught. This is for the one built at run -time -- from a plugin entry point, from configuration -- where there was -no type to check. -""" +Tables = Mapping[type[MetadataEntity], Mapping[str, type[MetadataEntity]]] +"""By kind, then by the identifier each entity is registered under.""" @dataclass(frozen=True, slots=True) class Context: """The entities in scope while metadata is being read. - Passed to every `coerce`, and most entities ignore it: a `gzip` codec - is a `gzip` codec whatever else is in scope. The ones that do not - ignore it hold other entities inside their own configuration -- a - `struct` data type holds field data types, a `sharding_indexed` codec - holds two codec pipelines -- and cannot read those without knowing - what is in scope inside them. - - A scope is not a property of the entities, it is a choice the reader - makes: judging against the specification alone, or against the - specification plus what `zarr-extensions` registers. + Built from classes with `Context.of`; extended with more by + `extended_with`. What each class is registered as is read off it -- + its kind is its base class, its key is its `identifier` -- so there + is nothing to misfile. """ - entities: EntityTables - - def __post_init__(self) -> None: - """Refuse a table an entity does not belong in. + tables: Tables - Two ways it can fail to. The entity may be of the wrong kind for - the point -- a codec under `data_type` -- which `EntityTables` - catches in source and this catches in a scope assembled at run - time. Or its key may not be its identifier. - - `resolve` finds a candidate by key and then asks the entity - whether the name is really one of its own, so a key that is not - the entity's `identifier` can never resolve. If the two disagree - -- a typo, or a rename that missed one of the two places the name - is written -- registration appears to succeed, validation runs, - and the verdict is clean. Indistinguishable from extension - openness, and the easiest way to ship a broken extension. - - The key is the identifier, not a name a document writes: the - raw-bytes family registers under an invented one that `accepts` - deliberately refuses. - """ - for field, table in self.tables().items(): - for key, entity in table.items(): - if not issubclass(entity, _ENTITY_KINDS[field]): - point = entity.extension_point - msg = ( - f"{entity.__name__} is registered at {field!r}, which takes " - f"{_ENTITY_KINDS[field].__name__} entities" - + (f"; register it at {point!r}" if point is not None else "") - ) - raise TypeError(msg) - if key != entity.identifier: - msg = ( - f"{entity.__name__} is registered at {field!r} under {key!r} " - f"but its identifier is {entity.identifier!r}; key the table by " - f"{entity.__name__}.identifier" - ) - raise ValueError(msg) - if inspect.isabstract(entity): - # What a kind leaves abstract -- `transition`, `grid`, - # `fill_value_problems` -- the entity answers, or it - # is not one this scope can use. - left = ", ".join(sorted(entity.__abstractmethods__)) - msg = ( - f"{entity.__name__} does not define {left}, which its base leaves " - "abstract; define it, if only to return the same thing as `incoming` or ()" - ) - raise TypeError(msg) - if "__dataclass_fields__" not in vars(entity) and any( - not is_class_var(annotation) for annotation in own_annotations(entity).values() - ): - # Class creation runs before `@dataclass` and cannot - # see whether it was applied; this is the next place - # the entity passes through before `coerce` builds it. - msg = ( - f"{entity.__name__} declares fields but is not a dataclass; decorate " - "it with @dataclass(frozen=True), which is what `coerce` builds it with" - ) - raise TypeError(msg) - - def extended_with(self, **entities: Unpack[PartialEntityTables]) -> Context: - """This scope, plus entities of your own at the points named. - - The merge is per point, so naming `codecs` adds codecs rather - than replacing the ones already in scope. A name already - registered is taken over by what is passed here, which is how a - reader substitutes its own reading of a codec the package - already models. - """ - return Context( - { - DATA_TYPE: {**self.entities["data_type"], **entities.get("data_type", {})}, - CODECS: {**self.entities["codecs"], **entities.get("codecs", {})}, - CHUNK_GRID: {**self.entities["chunk_grid"], **entities.get("chunk_grid", {})}, - CHUNK_KEY_ENCODING: { - **self.entities["chunk_key_encoding"], - **entities.get("chunk_key_encoding", {}), - }, - STORAGE_TRANSFORMERS: { - **self.entities["storage_transformers"], - **entities.get("storage_transformers", {}), - }, - } + @classmethod + def of(cls, *entities: type[MetadataEntity]) -> Context: + """A scope of exactly these entities; a later one takes over an identifier from an earlier.""" + tables: dict[type[MetadataEntity], dict[str, type[MetadataEntity]]] = { + kind: {} for kind in KINDS + } + for entity in entities: + kind = _registrable(entity) + tables[kind][entity.identifier] = entity + return cls( + MappingProxyType({kind: MappingProxyType(table) for kind, table in tables.items()}) ) - def tables(self) -> Mapping[ExtensionPointField, Mapping[str, type[MetadataEntity]]]: - """Every point's table, under the one kind all entities share. + def extended_with(self, *entities: type[MetadataEntity]) -> Context: + """This scope, plus entities of your own. - `entities` gives each point its own entity kind, which is the - point of it, and a key that is not a literal loses that. So the - widening is written out here, once, by hand rather than asserted - with a `cast`: reading each member by its own key is what makes - the result checked rather than promised. Anything that asks the - scope what is in it, rather than asking it about one point, wants - this. + A name already registered under the same kind is taken over by + what is passed here, which is how a reader substitutes its own + reading of a codec the package already models. """ - return { - DATA_TYPE: self.entities["data_type"], - CODECS: self.entities["codecs"], - CHUNK_GRID: self.entities["chunk_grid"], - CHUNK_KEY_ENCODING: self.entities["chunk_key_encoding"], - STORAGE_TRANSFORMERS: self.entities["storage_transformers"], - } - - @overload - def resolve(self, field: Literal["data_type"], name: str) -> type[DataTypeEntity] | None: ... + return Context.of(*self.entities(), *entities) - @overload - def resolve(self, field: Literal["codecs"], name: str) -> type[CodecEntity] | None: ... + def entities(self) -> tuple[type[MetadataEntity], ...]: + """Every entity in scope, kind by kind.""" + return tuple(entity for table in self.tables.values() for entity in table.values()) - @overload - def resolve(self, field: Literal["chunk_grid"], name: str) -> type[ChunkGridEntity] | None: ... + def resolve(self, kind: type[_EntityT], name: str) -> type[_EntityT] | None: + """The entity of `kind` that `name` denotes, or None if out of scope. - @overload - def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: ... - - def resolve(self, field: ExtensionPointField, name: str) -> type[MetadataEntity] | None: - """The entity `name` denotes at `field`, or None if out of scope. - - Out of scope is not an error: an unknown name may be an extension - this reader does not model, and openness means leaving it unjudged. + `kind` may be a subclass of a kind -- `GzipCodec`, a family -- in + which case only an entity under it resolves. Out of scope is not + an error: an unknown name may be an extension this reader does + not model, and openness means leaving it unjudged. The entity has the last word, via `accepts`. A name that is a key still has to be claimed, because a family's key is an invented identifier that no document may write; and a name that is not a key may still belong to a family, which is what the scan is for. - A third party registers one the same way, with no table of - spellings anywhere in this package. """ - table = self.tables()[field] + registered = kind_of(kind) + if registered is None: + return None + table = self.tables.get(registered, {}) entity = table.get(name) - if entity is not None: - return entity if entity.accepts(name) else None - # A family covers many names with one class, so its entry cannot - # be keyed by all of them; it is keyed by an invented identifier - # and recognizes its own. Asked only when the name is not a key, - # so the common case stays a lookup. First match wins, and two - # entities claiming one name is a scope that contradicts itself. - return next((candidate for candidate in table.values() if candidate.accepts(name)), None) - - @overload - def coerce( - self, - field: Literal["data_type"], - value: object, - loc: Loc = (), - *, - envelope_judged: bool = False, - ) -> tuple[DataTypeEntity | Opaque, tuple[ValidationProblem, ...]]: ... - - @overload - def coerce( - self, - field: Literal["codecs"], - value: object, - loc: Loc = (), - *, - envelope_judged: bool = False, - ) -> tuple[CodecEntity | Opaque, tuple[ValidationProblem, ...]]: ... - - @overload - def coerce( - self, - field: Literal["chunk_grid"], - value: object, - loc: Loc = (), - *, - envelope_judged: bool = False, - ) -> tuple[ChunkGridEntity | Opaque, tuple[ValidationProblem, ...]]: ... - - @overload - def coerce( - self, - field: ExtensionPointField, - value: object, - loc: Loc = (), - *, - envelope_judged: bool = False, - ) -> tuple[MetadataEntity | Opaque, tuple[ValidationProblem, ...]]: ... + if entity is not None and not entity.accepts(name): + return None + if entity is None: + # A family covers many names with one class, so its entry + # cannot be keyed by all of them; it is keyed by an invented + # identifier and recognizes its own. Asked only when the name + # is not a key, so the common case stays a lookup. First + # match wins, and two entities claiming one name is a scope + # that contradicts itself. + entity = next( + (candidate for candidate in table.values() if candidate.accepts(name)), None + ) + if entity is None or not issubclass(entity, kind): + return None + return entity def coerce( self, - field: ExtensionPointField, + kind: type[_EntityT], value: object, loc: Loc = (), *, envelope_judged: bool = False, - ) -> tuple[MetadataEntity | Opaque, tuple[ValidationProblem, ...]]: - """One nested entity, read in this scope. + ) -> tuple[_EntityT | Opaque, tuple[ValidationProblem, ...]]: + """One nested entity of `kind`, read in this scope. The primitive the containing entities are built from: a `struct` data type reads its fields with it, a `sharding_indexed` codec its @@ -372,7 +200,7 @@ def coerce( *problems, ValidationProblem(loc, f"expected a metadata field, got {value!r}", "invalid_type"), ) - entity_type = self.resolve(field, name) + entity_type = self.resolve(kind, name) if entity_type is None: return Opaque(value, "out_of_scope"), tuple(problems) entity, found = entity_type.coerce(value, self) @@ -384,85 +212,81 @@ def coerce( return entity, tuple(problems) -_CORE_CODECS: Final[dict[str, type[CodecEntity]]] = { - BloscCodec.identifier: BloscCodec, - BytesCodec.identifier: BytesCodec, - Crc32cCodec.identifier: Crc32cCodec, - GzipCodec.identifier: GzipCodec, - ShardingIndexedCodec.identifier: ShardingIndexedCodec, - TransposeCodec.identifier: TransposeCodec, -} -_EXTENSION_CODECS: Final[dict[str, type[CodecEntity]]] = { - CastValueCodec.identifier: CastValueCodec, - ScaleOffsetCodec.identifier: ScaleOffsetCodec, - ZstdCodec.identifier: ZstdCodec, -} - -_CORE_DATA_TYPES: Final[dict[str, type[DataTypeEntity]]] = { - BoolDataType.identifier: BoolDataType, - Int8DataType.identifier: Int8DataType, - Int16DataType.identifier: Int16DataType, - Int32DataType.identifier: Int32DataType, - Int64DataType.identifier: Int64DataType, - Uint8DataType.identifier: Uint8DataType, - Uint16DataType.identifier: Uint16DataType, - Uint32DataType.identifier: Uint32DataType, - Uint64DataType.identifier: Uint64DataType, - Float16DataType.identifier: Float16DataType, - Float32DataType.identifier: Float32DataType, - Float64DataType.identifier: Float64DataType, - Complex64DataType.identifier: Complex64DataType, - Complex128DataType.identifier: Complex128DataType, - RawBytesDataType.identifier: RawBytesDataType, -} -_EXTENSION_DATA_TYPES: Final[dict[str, type[DataTypeEntity]]] = { - BytesDataType.identifier: BytesDataType, - StringDataType.identifier: StringDataType, - NumpyDatetime64DataType.identifier: NumpyDatetime64DataType, - NumpyTimedelta64DataType.identifier: NumpyTimedelta64DataType, - StructDataType.identifier: StructDataType, -} - -_CORE_CHUNK_GRIDS: Final[dict[str, type[ChunkGridEntity]]] = { - RegularChunkGrid.identifier: RegularChunkGrid, -} -_EXTENSION_CHUNK_GRIDS: Final[dict[str, type[ChunkGridEntity]]] = { - RectilinearChunkGrid.identifier: RectilinearChunkGrid, -} - -_CORE_CHUNK_KEY_ENCODINGS: Final[dict[str, type[MetadataEntity]]] = { - DefaultChunkKeyEncoding.identifier: DefaultChunkKeyEncoding, - V2ChunkKeyEncoding.identifier: V2ChunkKeyEncoding, -} - - -CORE: Final = Context( - { - CODECS: _CORE_CODECS, - DATA_TYPE: _CORE_DATA_TYPES, - CHUNK_GRID: _CORE_CHUNK_GRIDS, - CHUNK_KEY_ENCODING: _CORE_CHUNK_KEY_ENCODINGS, - STORAGE_TRANSFORMERS: {}, - } -) -"""Only what the Zarr v3 specification defines.""" +def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: + """The kind `entity` is registered under; `TypeError` for a class no scope can use. -CORE_AND_EXTENSIONS: Final = Context( - { - CODECS: {**_CORE_CODECS, **_EXTENSION_CODECS}, - DATA_TYPE: {**_CORE_DATA_TYPES, **_EXTENSION_DATA_TYPES}, - CHUNK_GRID: {**_CORE_CHUNK_GRIDS, **_EXTENSION_CHUNK_GRIDS}, - CHUNK_KEY_ENCODING: {**_CORE_CHUNK_KEY_ENCODINGS}, - STORAGE_TRANSFORMERS: {}, - } + Class creation refuses what it can see; these are the things it + cannot -- the decorator, what a kind leaves abstract -- checked at + the first place the class passes through before `coerce` builds it. + """ + kind = kind_of(entity) + if kind is None: + msg = ( + f"{entity.__name__} is of no kind; subclass a codec kind, DataTypeEntity, " + "ChunkGridEntity, ChunkKeyEncodingEntity or StorageTransformerEntity" + ) + raise TypeError(msg) + if inspect.isabstract(entity): + left = ", ".join(sorted(entity.__abstractmethods__)) + msg = ( + f"{entity.__name__} does not define {left}, which its base leaves abstract; " + "define it, if only to return the same thing as `incoming` or ()" + ) + raise TypeError(msg) + if "__dataclass_fields__" not in vars(entity) and any( + not is_class_var(annotation) for annotation in own_annotations(entity).values() + ): + msg = ( + f"{entity.__name__} declares fields but is not a dataclass; decorate it with " + "@dataclass(frozen=True), which is what `coerce` builds it with" + ) + raise TypeError(msg) + return kind + + +_CORE: Final[tuple[type[MetadataEntity], ...]] = ( + BloscCodec, + BytesCodec, + Crc32cCodec, + GzipCodec, + ShardingIndexedCodec, + TransposeCodec, + BoolDataType, + Int8DataType, + Int16DataType, + Int32DataType, + Int64DataType, + Uint8DataType, + Uint16DataType, + Uint32DataType, + Uint64DataType, + Float16DataType, + Float32DataType, + Float64DataType, + Complex64DataType, + Complex128DataType, + RawBytesDataType, + RegularChunkGrid, + DefaultChunkKeyEncoding, + V2ChunkKeyEncoding, ) -"""What the specification defines, plus what `zarr-extensions` registers.""" +"""What the Zarr v3 specification itself defines.""" + +_EXTENSIONS: Final[tuple[type[MetadataEntity], ...]] = ( + CastValueCodec, + ScaleOffsetCodec, + ZstdCodec, + BytesDataType, + StringDataType, + NumpyDatetime64DataType, + NumpyTimedelta64DataType, + StructDataType, + RectilinearChunkGrid, +) +"""What `zarr-extensions` registers and this package models.""" +CORE: Final = Context.of(*_CORE) +"""Only what the Zarr v3 specification defines.""" -__all__ = [ - "CORE", - "CORE_AND_EXTENSIONS", - "Context", - "EntityTables", - "PartialEntityTables", -] +CORE_AND_EXTENSIONS: Final = Context.of(*_CORE, *_EXTENSIONS) +"""What the specification defines, plus what `zarr-extensions` registers.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 0199fc61a4..d486864da3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -14,7 +14,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( - MetadataEntity, + ChunkKeyEncodingEntity, ) DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" @@ -71,7 +71,7 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class DefaultChunkKeyEncoding(MetadataEntity[DefaultChunkKeyEncodingMetadata]): +class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity[DefaultChunkKeyEncodingMetadata]): """The `default` chunk key encoding, coerced from its metadata.""" separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 57e5ed0749..87cac6d1ae 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -20,7 +20,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( - MetadataEntity, + ChunkKeyEncodingEntity, ) V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" @@ -77,7 +77,7 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class V2ChunkKeyEncoding(MetadataEntity[V2ChunkKeyEncodingMetadata]): +class V2ChunkKeyEncoding(ChunkKeyEncodingEntity[V2ChunkKeyEncodingMetadata]): """The `v2` chunk key encoding, coerced from its metadata.""" separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 3c90a31f75..b80d964f39 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -28,18 +28,18 @@ being judged rather than demanded. `zarr_metadata.rules.validate_array_metadata_v3(document, context=...)` returns a tuple of `ValidationProblem(loc, message, kind)`, `kind` one of `ProblemKind`, each `loc` indexing into the document: -`("codecs", 1, "configuration", "level")`. `SCOPE.coerce("codecs", entry)` -reads one metadata field and returns `(entity, problems)` where `entity` -is the entity or an `Opaque` -- never `None` -- with `loc` relative to the -entry: `("configuration", "level")`. An entity's own +`("codecs", 1, "configuration", "level")`. `SCOPE.coerce(CodecEntity, entry)` +reads one metadata field as an entity of that kind and returns +`(entity, problems)` where `entity` is the entity or an `Opaque` -- never +`None` -- with `loc` relative to the entry: `("configuration", "level")`. An entity's own `coerce(value, context)` returns `(entity or None, problems)`; that is `Coerced`. Constructing an entity by hand raises `MetadataValidationError` with `loc` relative to the configuration: `("level",)`. **Writing an extension.** Subclass the kind of thing it is -- a codec's kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), -`DataTypeEntity`, `ChunkGridEntity`, or `MetadataEntity` for the two -points that take anything; declare the configuration as dataclass +`DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity` or +`StorageTransformerEntity`; declare the configuration as dataclass fields; put every rule finer than a type in `__post_init__`; add the class to a scope. Complete, and runnable as written: @@ -71,7 +71,7 @@ def __post_init__(self) -> None: ) ) - SCOPE = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}) + SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec) validate_array_metadata_v3(document, context=SCOPE) The fields are the only place the shape is written. Which members exist, @@ -127,9 +127,12 @@ def __post_init__(self) -> None: What a kind leaves abstract, registration refuses an entity for not defining; the other mistakes an author would not otherwise see -- a `scalar_storage` outside the listed values, a nested field without -`Opaque`, a class without `@dataclass`, a codec subclassing `CodecEntity` -instead of a kind -- are refused at class creation or registration with -a message that says what to write. +`Opaque`, a class without `@dataclass`, an entity subclassing +`MetadataEntity` or `CodecEntity` instead of a kind -- are refused at +class creation or registration with a message that says what to write. +A scope reads what a class is off the class: its kind is its base, its +key is its `identifier`, so `extended_with` takes the classes and nothing +can be misfiled. **Naming the JSON type.** `CodecEntity[AcmeLz4Metadata]` types `to_json` as your own TypedDict rather than as any metadata field. The shape is @@ -148,7 +151,7 @@ def __post_init__(self) -> None: A name in no scope is not rejected -- that is what extension openness means -- so registering yours is how you get it judged rather than waved through. `CORE` is what the specification defines; `CORE_AND_EXTENSIONS` -adds the `zarr-extensions` registry; `extended_with` adds yours. +adds the `zarr-extensions` registry; `extended_with(*classes)` adds yours. One known friction, under mypy only. An entity's `to_json` returns its own object TypedDict, and mypy does not accept that where a @@ -173,32 +176,28 @@ def __post_init__(self) -> None: from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._document import ArrayDocumentV3 from zarr_metadata.v3._entity import ( - CHUNK_GRID, - CHUNK_KEY_ENCODING, - CODECS, - DATA_TYPE, FROM_NAME, - STORAGE_TRANSFORMERS, ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, ChunkGridEntity, + ChunkKeyEncodingEntity, CodecEntity, CodecKind, Coerced, DataTypeEntity, - ExtensionPointField, Loc, MetadataEntity, Opaque, StorageClass, + StorageTransformerEntity, is_integer, named_configuration, problem, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid, Extents -from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context, EntityTables +from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context from zarr_metadata.v3.data_type._families import ( ComplexDataType, FloatDataType, @@ -207,14 +206,9 @@ def __post_init__(self) -> None: ) __all__ = [ - "CHUNK_GRID", - "CHUNK_KEY_ENCODING", - "CODECS", "CORE", "CORE_AND_EXTENSIONS", - "DATA_TYPE", "FROM_NAME", - "STORAGE_TRANSFORMERS", "UNSET", "ArrayArrayCodec", "ArrayBytesCodec", @@ -223,14 +217,13 @@ def __post_init__(self) -> None: "BytesBytesCodec", "ChunkGrid", "ChunkGridEntity", + "ChunkKeyEncodingEntity", "CodecEntity", "CodecKind", "Coerced", "ComplexDataType", "Context", "DataTypeEntity", - "EntityTables", - "ExtensionPointField", "Extents", "FloatDataType", "IntegerDataType", @@ -242,6 +235,7 @@ def __post_init__(self) -> None: "Opaque", "ProblemKind", "StorageClass", + "StorageTransformerEntity", "ValidationProblem", "ZarrV3MetadataFieldJSON", "chain_problems", diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py index 87fb998040..cd4ea4d6d8 100644 --- a/packages/zarr-metadata/tests/rules/test_chain_properties.py +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -27,7 +27,7 @@ valid_documents, ) from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.v3.entity import CODECS, CORE_AND_EXTENSIONS +from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS, CodecEntity if TYPE_CHECKING: from collections.abc import Mapping @@ -61,7 +61,7 @@ def test_the_strategies_cover_every_codec_the_package_models() -> None: for kinds, expected in ((ARRAY_ARRAY, "array_array"), (ARRAY_BYTES, "array_bytes")): for entry in kinds: name = entry.__annotations__["name"].__args__[0] - entity = CORE_AND_EXTENSIONS.resolve(CODECS, name) + entity = CORE_AND_EXTENSIONS.resolve(CodecEntity, name) assert entity is not None, name assert entity.kind == expected diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index 435e665120..ab7d572569 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -10,6 +10,7 @@ from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._parts import ChunkGrid, shard_index_grid from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS +from zarr_metadata.v3.entity import ChunkGridEntity BASE: Mapping[str, object] = { "zarr_format": 3, @@ -99,7 +100,9 @@ def _grid_of(grid: object, shape: object) -> ChunkGrid: if isinstance(grid, Mapping) else None ) - entity_type = CORE_AND_EXTENSIONS.resolve("chunk_grid", name) if isinstance(name, str) else None + entity_type = ( + CORE_AND_EXTENSIONS.resolve(ChunkGridEntity, name) if isinstance(name, str) else None + ) if entity_type is None: return ChunkGrid.unreadable(shape) entity, _ = entity_type.coerce(grid, CORE_AND_EXTENSIONS) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index ba037658da..cba836eb57 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -294,14 +294,12 @@ def test_all_is_grouped_and_unique() -> None: "StorageClass", "Loc", "Extents", - "ExtensionPointField", "Context", "Coerced", "ChunkGrid", "ArrayParts", "ArrayDocumentV3", "Endianness", - "EntityTables", "Invalid", "HexFloat16", "HexFloat32", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 9eed647658..4f49ffc41f 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -97,7 +97,7 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: ) -SCOPE = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeAffineCodec.identifier: AcmeAffineCodec}) +SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeAffineCodec) BYTES_LE = {"name": "bytes", "configuration": {"endian": "little"}} diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index f8215e2623..3a943d0e98 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -151,9 +151,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP from zarr_metadata.rules import canonicalize_array_metadata_v3, validate_array_metadata_v3 from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS, ArrayDocumentV3, Context, Opaque -SCOPE: Context = CORE_AND_EXTENSIONS.extended_with( - data_type={AcmeDecimalDataType.identifier: AcmeDecimalDataType} -) +SCOPE: Context = CORE_AND_EXTENSIONS.extended_with(AcmeDecimalDataType) LITTLE_ENDIAN_BYTES = {"name": "bytes", "configuration": {"endian": "little"}} diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index b929746e62..b8264cebcf 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -75,11 +75,28 @@ from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType -from zarr_metadata.v3.entity import ArrayDocumentV3, ExtensionPointField, MetadataEntity +from zarr_metadata.v3.entity import ( + ArrayDocumentV3, + ChunkGridEntity, + ChunkKeyEncodingEntity, + CodecEntity, + DataTypeEntity, + MetadataEntity, + StorageTransformerEntity, +) # Every registered entity, keyed by `:` -- an identifier # is unique only within its extension point, and `bytes` is both a codec # and a data type. +# The field of a v3 array document each kind is read from: the test ids. +POINT: dict[type[MetadataEntity], str] = { + DataTypeEntity: "data_type", + ChunkGridEntity: "chunk_grid", + ChunkKeyEncodingEntity: "chunk_key_encoding", + CodecEntity: "codecs", + StorageTransformerEntity: "storage_transformers", +} + ENTITIES: dict[str, type[MetadataEntity]] = { "codecs:blosc": BloscCodec, "codecs:bytes": BytesCodec, @@ -272,8 +289,8 @@ def test_to_json_conforms_across_a_valid_document(document: dict[str, object]) - def test_every_registered_entity_is_checked_here() -> None: registered = { - f"{field}:{identifier}" - for field, entities in CORE_AND_EXTENSIONS.tables().items() + f"{POINT[kind]}:{identifier}" + for kind, entities in CORE_AND_EXTENSIONS.tables.items() for identifier in entities } assert registered == set(ENTITIES) @@ -281,15 +298,15 @@ def test_every_registered_entity_is_checked_here() -> None: def test_core_is_a_subset_of_core_and_extensions() -> None: - both = CORE_AND_EXTENSIONS.tables() - for field, entities in CORE.tables().items(): - assert entities.items() <= both[field].items() + both = CORE_AND_EXTENSIONS.tables + for kind, entities in CORE.tables.items(): + assert entities.items() <= both[kind].items() def test_a_name_out_of_scope_resolves_to_nothing() -> None: # Not an error: an unmodelled extension is left unjudged, not rejected. - assert CORE.resolve("codecs", "mycorp.secret") is None - assert CORE.resolve("codecs", "blosc") is BloscCodec + assert CORE.resolve(CodecEntity, "mycorp.secret") is None + assert CORE.resolve(CodecEntity, "blosc") is BloscCodec def test_an_entity_round_trips_through_its_json_form() -> None: @@ -516,23 +533,23 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: # Entities whose written form and canonical form differ, or could. -FAITHFUL: dict[str, tuple[ExtensionPointField, object]] = { +FAITHFUL: dict[str, tuple[type[MetadataEntity], object]] = { "rectilinear-expanded": ( - "chunk_grid", + ChunkGridEntity, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": ((32, 32, 32),)}, }, ), "rectilinear-encoded": ( - "chunk_grid", + ChunkGridEntity, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": (((32, 3),),)}, }, ), "blosc-ignored-typesize": ( - "codecs", + CodecEntity, { "name": "blosc", "configuration": { @@ -544,13 +561,13 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: }, }, ), - "raw-bytes-padded": ("data_type", "r008"), + "raw-bytes-padded": (DataTypeEntity, "r008"), "scale-offset-scalar": ( - "codecs", + CodecEntity, {"name": "scale_offset", "configuration": {"offset": 2, "scale": 0.5}}, ), "struct-nested": ( - "data_type", + DataTypeEntity, { "name": "struct", "configuration": { @@ -562,7 +579,7 @@ def test_an_unreadable_member_is_not_judged_by_its_default() -> None: @pytest.mark.parametrize(("field", "written"), FAITHFUL.values(), ids=list(FAITHFUL)) -def test_to_json_writes_back_what_was_read(field: ExtensionPointField, written: object) -> None: +def test_to_json_writes_back_what_was_read(field: type[MetadataEntity], written: object) -> None: # Serialization is not canonicalization. A reader that reads a # document and writes it back must not change bytes it was not asked # to change -- `canonical()` is where you ask. @@ -574,7 +591,7 @@ def test_to_json_writes_back_what_was_read(field: ExtensionPointField, written: def test_canonical_is_what_simplifies() -> None: encoded, _ = CORE_AND_EXTENSIONS.coerce( - "chunk_grid", + ChunkGridEntity, { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": ((32, 32, 32),)}, @@ -586,7 +603,7 @@ def test_canonical_is_what_simplifies() -> None: "configuration": {"kind": "inline", "chunk_shapes": (((32, 3),),)}, } blosc, _ = CORE_AND_EXTENSIONS.coerce( - "codecs", + CodecEntity, { "name": "blosc", "configuration": { @@ -604,7 +621,7 @@ def test_canonical_is_what_simplifies() -> None: def test_canonical_reaches_a_contained_entity() -> None: shard, _ = CORE_AND_EXTENSIONS.coerce( - "codecs", + CodecEntity, { "name": "sharding_indexed", "configuration": { @@ -635,7 +652,7 @@ def test_error_an_explicit_null_scalar_is_refused() -> None: # `null` is a value the document wrote, distinct from absence -- and # no data type admits it as a scalar, so the codec cannot be built. codec, problems = CORE_AND_EXTENSIONS.coerce( - "codecs", {"name": "scale_offset", "configuration": {"offset": None}} + CodecEntity, {"name": "scale_offset", "configuration": {"offset": None}} ) assert codec is not None assert not isinstance(codec, MetadataEntity) @@ -643,20 +660,20 @@ def test_error_an_explicit_null_scalar_is_refused() -> None: # (an entity whose configuration holds a mutable JSON value) -MUTABLE_MEMBERS: dict[str, tuple[ExtensionPointField, object]] = { +MUTABLE_MEMBERS: dict[str, tuple[type[MetadataEntity], object]] = { "scale-offset-object": ( - "codecs", + CodecEntity, {"name": "scale_offset", "configuration": {"offset": {"a": 1}}}, ), "cast-value-scalar-map": ( - "codecs", + CodecEntity, { "name": "cast_value", "configuration": {"data_type": "int8", "scalar_map": {"encode": (("NaN", 0),)}}, }, ), "struct-fields": ( - "data_type", + DataTypeEntity, {"name": "struct", "configuration": {"fields": ({"name": "a", "data_type": "uint8"},)}}, ), } @@ -664,7 +681,7 @@ def test_error_an_explicit_null_scalar_is_refused() -> None: @pytest.mark.parametrize(("field", "written"), MUTABLE_MEMBERS.values(), ids=list(MUTABLE_MEMBERS)) def test_to_json_shares_no_mutable_state_with_the_entity( - field: ExtensionPointField, written: object + field: type[MetadataEntity], written: object ) -> None: # The model layer has this test; the entity layer did not, and handed # out its own dict -- so a caller mutating the document it was given @@ -701,7 +718,7 @@ def test_a_member_the_entity_does_not_model_is_not_written_back() -> None: "typo_key": 1, }, } - codec, problems = CORE_AND_EXTENSIONS.coerce("codecs", entry) + codec, problems = CORE_AND_EXTENSIONS.coerce(CodecEntity, entry) assert [(p.loc, p.kind) for p in problems] == [(("configuration", "typo_key"), "unknown_key")] assert isinstance(codec, BloscCodec) assert "typo_key" not in codec.to_json()["configuration"] @@ -730,7 +747,7 @@ def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: # A storage transformer: the one extension point nothing in the package # models, so the only way to reach it is to register one. @dataclasses.dataclass(frozen=True) -class AcmeShardCache(MetadataEntity): +class AcmeShardCache(StorageTransformerEntity): """A third-party storage transformer with a member canonical form drops.""" verbose: bool | UNSET = UNSET @@ -745,9 +762,7 @@ def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> # `to_json` is faithful, entities included; `canonical` walks every # field that holds an entity -- `storage_transformers` among them, # which the hand-written walk it replaces never reached. - scope = CORE_AND_EXTENSIONS.extended_with( - storage_transformers={AcmeShardCache.identifier: AcmeShardCache} - ) + scope = CORE_AND_EXTENSIONS.extended_with(AcmeShardCache) document = { "zarr_format": 3, "node_type": "array", diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 27001ee5bf..6549cb21bd 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -21,7 +21,6 @@ from zarr_metadata.v3.codec.blosc import BloscCodec from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecObject from zarr_metadata.v3.entity import ( - CORE, CORE_AND_EXTENSIONS, FROM_NAME, ArrayArrayCodec, @@ -29,6 +28,7 @@ ArrayParts, BytesBytesCodec, ChunkGridEntity, + ChunkKeyEncodingEntity, CodecEntity, CodecKind, Context, @@ -78,10 +78,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP def _scope() -> Context: - return CORE_AND_EXTENSIONS.extended_with( - codecs={AcmeLz4Codec.identifier: AcmeLz4Codec}, - data_type={AcmeFloat8DataType.identifier: AcmeFloat8DataType}, - ) + return CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec, AcmeFloat8DataType) SCOPE = _scope() @@ -160,33 +157,15 @@ class Nameless(BytesBytesCodec): """A codec that forgot to say what it is.""" -def test_error_a_registry_key_must_be_the_identifier() -> None: - # Otherwise `resolve` never finds it and the document is silently - # waved through, indistinguishable from openness. - with pytest.raises(ValueError, match="registered at 'codecs' under 'acme.lz-4'"): - CORE.extended_with(codecs={"acme.lz-4": AcmeLz4Codec}) - - -def test_error_an_entity_cannot_be_registered_at_the_wrong_point() -> None: - # `EntityTables` says so to the type checker, which settles a scope - # written out in source. A scope assembled at run time -- from an - # entry point, from configuration -- had no type to check, and a - # codec under `data_type` would resolve and then be asked for a - # storage class it has no answer to. - with pytest.raises(TypeError, match="registered at 'data_type', which takes DataTypeEntity"): - # Deliberately wrong, and pyright says so; the runtime refusal is what is under test. - CORE.extended_with(data_type={AcmeLz4Codec.identifier: AcmeLz4Codec}) # pyright: ignore[reportArgumentType] - - def test_the_entity_layer_answers_what_a_reader_needs() -> None: # The questions zarr-python asks before it can read a chunk. - data_type, problems = CORE_AND_EXTENSIONS.coerce("data_type", "int32") + data_type, problems = CORE_AND_EXTENSIONS.coerce(DataTypeEntity, "int32") assert problems == () assert isinstance(data_type, DataTypeEntity) assert data_type.storage_class() == "multi_byte" grid, problems = CORE_AND_EXTENSIONS.coerce( - "chunk_grid", {"name": "regular", "configuration": {"chunk_shape": (32, 32)}} + ChunkGridEntity, {"name": "regular", "configuration": {"chunk_shape": (32, 32)}} ) assert problems == () assert isinstance(grid, ChunkGridEntity) @@ -254,7 +233,7 @@ def test_every_extension_point_is_an_exhaustive_two_case_union() -> None: array = ArrayDocumentV3.from_json(_document(data_type="mycorp.decimal", fill_value=0)) assert isinstance(array.data_type, (DataTypeEntity, Opaque)) assert isinstance(array.chunk_grid, (ChunkGridEntity, Opaque)) - assert isinstance(array.chunk_key_encoding, (MetadataEntity, Opaque)) + assert isinstance(array.chunk_key_encoding, (ChunkKeyEncodingEntity, Opaque)) assert all(isinstance(codec, (CodecEntity, Opaque)) for codec in array.codecs) @@ -325,19 +304,17 @@ def test_a_third_party_can_register_a_family() -> None: # holds a table of spellings: the entity registers under an invented # identifier and `resolve` asks it, so a family is registered exactly # like a single name. - scope = CORE_AND_EXTENSIONS.extended_with( - data_type={AcmeFixedDataType.identifier: AcmeFixedDataType} - ) + scope = CORE_AND_EXTENSIONS.extended_with(AcmeFixedDataType) for name in ("acme.fixed8", "acme.fixed128"): - assert scope.resolve("data_type", name) is AcmeFixedDataType - entity, problems = scope.coerce("data_type", name) + assert scope.resolve(DataTypeEntity, name) is AcmeFixedDataType + entity, problems = scope.coerce(DataTypeEntity, name) assert problems == () assert isinstance(entity, AcmeFixedDataType) assert entity.to_json() == name # The invented identifier is not a name a document may write, and a # near-miss is still nobody's. - assert scope.resolve("data_type", AcmeFixedDataType.identifier) is None - assert scope.resolve("data_type", "acme.fixed") is None + assert scope.resolve(DataTypeEntity, AcmeFixedDataType.identifier) is None + assert scope.resolve(DataTypeEntity, "acme.fixed") is None def test_error_a_member_needs_a_check_from_somewhere() -> None: @@ -368,14 +345,12 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non # `inner: CodecEntity | Opaque` is the whole declaration. Reading it # in scope, writing it back, and canonicalizing through it all follow # from the annotation, so a wrapper is as short to write as a leaf. - scope = CORE_AND_EXTENSIONS.extended_with( - codecs={AcmeWrapperCodec.identifier: AcmeWrapperCodec} - ) + scope = CORE_AND_EXTENSIONS.extended_with(AcmeWrapperCodec) entry = { "name": "acme.wrapper", "configuration": {"inner": {"name": "gzip", "configuration": {"level": 5}}}, } - codec, problems = scope.coerce("codecs", entry) + codec, problems = scope.coerce(CodecEntity, entry) assert problems == () assert isinstance(codec, AcmeWrapperCodec) assert isinstance(codec.inner, GzipCodec) @@ -384,7 +359,7 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non # An inner codec the scope does not model stays verbatim, as anywhere. unknown = {"name": "acme.wrapper", "configuration": {"inner": "acme.unknown"}} - codec, problems = scope.coerce("codecs", unknown) + codec, problems = scope.coerce(CodecEntity, unknown) assert problems == () assert isinstance(codec, AcmeWrapperCodec) assert isinstance(codec.inner, Opaque) @@ -395,7 +370,7 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non "name": "acme.wrapper", "configuration": {"inner": {"name": "gzip", "configuration": {"level": 99}}}, } - _, problems = scope.coerce("codecs", bad) + _, problems = scope.coerce(CodecEntity, bad) assert [problem.loc for problem in problems] == [ ("configuration", "inner", "configuration", "level") ] @@ -416,7 +391,7 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non } }, } - codec, _ = scope.coerce("codecs", verbose) + codec, _ = scope.coerce(CodecEntity, verbose) assert isinstance(codec, AcmeWrapperCodec) inner = codec.canonical().inner assert isinstance(inner, BloscCodec) @@ -461,7 +436,7 @@ def simplified(self) -> Self: def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: # `MetadataEntity` is registered at no single point, so a field typed # as one could not be resolved through any scope. - with pytest.raises(TypeError, match="has no `extension_point`"): + with pytest.raises(TypeError, match="is of no kind; annotate it with a codec kind"): @dataclass(frozen=True) class Vague(BytesBytesCodec): @@ -512,11 +487,13 @@ def test_a_rule_about_a_member_is_post_init() -> None: # The rule runs on the typed members and reports relative to the # configuration; `coerce` catches what it raises and locates it in # the document, and the constructor raises it as it is. - scope = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeBlockCodec.identifier: AcmeBlockCodec}) - codec, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": 64}}) + scope = CORE_AND_EXTENSIONS.extended_with(AcmeBlockCodec) + codec, problems = scope.coerce( + CodecEntity, {"name": "acme.block", "configuration": {"block": 64}} + ) assert problems == () assert isinstance(codec, AcmeBlockCodec) - _, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": 6}}) + _, problems = scope.coerce(CodecEntity, {"name": "acme.block", "configuration": {"block": 6}}) assert [(p.loc, p.message) for p in problems] == [ (("configuration", "block"), "expected a power of two, got 6") ] @@ -525,7 +502,7 @@ def test_a_rule_about_a_member_is_post_init() -> None: AcmeBlockCodec(block=6) assert [p.loc for p in caught.value.problems] == [("block",)] # A member that failed its type check never reaches the rule. - _, problems = scope.coerce("codecs", {"name": "acme.block", "configuration": {"block": "x"}}) + _, problems = scope.coerce(CodecEntity, {"name": "acme.block", "configuration": {"block": "x"}}) assert [p.kind for p in problems] == ["invalid_type"] @@ -567,14 +544,16 @@ class AcmeScaled(ArrayArrayCodec): def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming - scope = CORE_AND_EXTENSIONS.extended_with(codecs={AcmeScaled.identifier: AcmeScaled}) + scope = CORE_AND_EXTENSIONS.extended_with(AcmeScaled) for written in (2, 2.5): codec, problems = scope.coerce( - "codecs", {"name": "acme.scaled", "configuration": {"scale": written}} + CodecEntity, {"name": "acme.scaled", "configuration": {"scale": written}} ) assert problems == () assert isinstance(codec, AcmeScaled) - _, problems = scope.coerce("codecs", {"name": "acme.scaled", "configuration": {"scale": True}}) + _, problems = scope.coerce( + CodecEntity, {"name": "acme.scaled", "configuration": {"scale": True}} + ) assert [(p.loc, p.message) for p in problems] == [ (("configuration", "scale"), "expected a number, got True") ] @@ -590,7 +569,7 @@ class Undecorated(BytesBytesCodec): identifier: ClassVar[str] = "acme.undecorated" with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): - CORE_AND_EXTENSIONS.extended_with(codecs={Undecorated.identifier: Undecorated}) + CORE_AND_EXTENSIONS.extended_with(Undecorated) def test_error_a_nested_field_admits_opaque() -> None: @@ -614,7 +593,7 @@ class Silent(ArrayArrayCodec): with pytest.raises( TypeError, match="does not define transition, which its base leaves abstract" ): - CORE_AND_EXTENSIONS.extended_with(codecs={Silent.identifier: Silent}) + CORE_AND_EXTENSIONS.extended_with(Silent) def test_error_a_codec_is_of_a_kind() -> None: @@ -636,7 +615,7 @@ class Lax(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "single_byte" with pytest.raises(TypeError, match="does not define fill_value_problems"): - CORE_AND_EXTENSIONS.extended_with(data_type={Lax.identifier: Lax}) + CORE_AND_EXTENSIONS.extended_with(Lax) def test_error_a_literal_class_variable_holds_a_listed_value() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py index eb7e762d55..89a3ee521c 100644 --- a/packages/zarr-metadata/tests/v3/test_fill_values.py +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -75,7 +75,7 @@ def _data_type(metadata: object) -> DataTypeEntity: name = metadata if isinstance(metadata, str) else entry_at(metadata, "name") assert isinstance(name, str), metadata - entity_type = CORE_AND_EXTENSIONS.resolve("data_type", name) + entity_type = CORE_AND_EXTENSIONS.resolve(DataTypeEntity, name) assert entity_type is not None, metadata entity, problems = entity_type.coerce(metadata, CORE_AND_EXTENSIONS) assert problems == (), problems @@ -98,7 +98,7 @@ def test_error_rejects(metadata: object, fill: object, reason: str) -> None: def test_error_a_malformed_raw_name_has_no_entity_to_ask() -> None: # `r12` is not a width, so the data type does not exist and there is # nothing to put a fill value to. - entity, problems = CORE_AND_EXTENSIONS.coerce("data_type", "r12") + entity, problems = CORE_AND_EXTENSIONS.coerce(DataTypeEntity, "r12") assert not isinstance(entity, DataTypeEntity) assert [problem.message for problem in problems] == [ "Expected 'r' where N is a positive multiple of 8, got 'r12'" @@ -107,4 +107,4 @@ def test_error_a_malformed_raw_name_has_no_entity_to_ask() -> None: def test_an_unmodelled_data_type_judges_nothing() -> None: # Extension openness: a fill value we cannot interpret is not wrong. - assert CORE_AND_EXTENSIONS.resolve("data_type", "mycorp.decimal") is None + assert CORE_AND_EXTENSIONS.resolve(DataTypeEntity, "mycorp.decimal") is None diff --git a/packages/zarr-metadata/tests/v3/test_resolve.py b/packages/zarr-metadata/tests/v3/test_resolve.py index 500e21229f..05bfdc3a1b 100644 --- a/packages/zarr-metadata/tests/v3/test_resolve.py +++ b/packages/zarr-metadata/tests/v3/test_resolve.py @@ -19,37 +19,36 @@ from zarr_metadata.v3.data_type.raw import RAW_BYTES_FAMILY, RawBytesDataType from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.entity import ( - CHUNK_GRID, - CODECS, CORE_AND_EXTENSIONS, - DATA_TYPE, - ExtensionPointField, + ChunkGridEntity, + CodecEntity, + DataTypeEntity, MetadataEntity, ) # (field, name, the entity that answers for it — None when nothing does) -RESOLUTIONS: dict[str, tuple[ExtensionPointField, str, type[MetadataEntity] | None]] = { - "plain-dtype": (DATA_TYPE, "uint8", Uint8DataType), - "dotted-dtype": (DATA_TYPE, "numpy.datetime64", NumpyDatetime64DataType), - "raw-8": (DATA_TYPE, "r8", RawBytesDataType), - "raw-24": (DATA_TYPE, "r24", RawBytesDataType), +RESOLUTIONS: dict[str, tuple[type[MetadataEntity], str, type[MetadataEntity] | None]] = { + "plain-dtype": (DataTypeEntity, "uint8", Uint8DataType), + "dotted-dtype": (DataTypeEntity, "numpy.datetime64", NumpyDatetime64DataType), + "raw-8": (DataTypeEntity, "r8", RawBytesDataType), + "raw-24": (DataTypeEntity, "r24", RawBytesDataType), # A malformed member reaches the family too: a misspelling of # something we model must be reported as such, not pass as an unknown # third-party extension. - "raw-not-multiple-of-8": (DATA_TYPE, "r12", RawBytesDataType), - "raw-zero": (DATA_TYPE, "r0", RawBytesDataType), + "raw-not-multiple-of-8": (DataTypeEntity, "r12", RawBytesDataType), + "raw-zero": (DataTypeEntity, "r0", RawBytesDataType), # Tables are per point, so the family cannot be reached from another. - "raw-shaped-codec-name": (CODECS, "r8", None), - "codec": (CODECS, "blosc", BloscCodec), - "unknown": (CODECS, "zfpy", None), + "raw-shaped-codec-name": (CodecEntity, "r8", None), + "codec": (CodecEntity, "blosc", BloscCodec), + "unknown": (CodecEntity, "zfpy", None), # The family's key is invented, so no document may write it. - "the-family-key-itself": (DATA_TYPE, RAW_BYTES_FAMILY, None), + "the-family-key-itself": (DataTypeEntity, RAW_BYTES_FAMILY, None), } @pytest.mark.parametrize(("field", "name", "expected"), RESOLUTIONS.values(), ids=list(RESOLUTIONS)) def test_a_name_resolves_to_the_entity_that_answers_for_it( - field: ExtensionPointField, name: str, expected: type[MetadataEntity] | None + field: type[MetadataEntity], name: str, expected: type[MetadataEntity] | None ) -> None: assert CORE_AND_EXTENSIONS.resolve(field, name) is expected @@ -60,15 +59,15 @@ def test_every_numeric_r_spelling_resolves_to_the_family(width: int) -> None: # the family claims a name by grammar shape, not by validity, so a # misspelled member of a family we model is reported as a misspelling # rather than passing as an unknown third-party extension. - assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, f"r{width}") is RawBytesDataType + assert CORE_AND_EXTENSIONS.resolve(DataTypeEntity, f"r{width}") is RawBytesDataType -OTHER_POINTS: tuple[ExtensionPointField, ...] = (CODECS, CHUNK_GRID) +OTHER_KINDS: tuple[type[MetadataEntity], ...] = (CodecEntity, ChunkGridEntity) -@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from(OTHER_POINTS)) +@given(width=st.integers(min_value=0, max_value=2**32), field=st.sampled_from(OTHER_KINDS)) def test_r_shaped_names_resolve_to_nothing_outside_data_types( - width: int, field: ExtensionPointField + width: int, field: type[MetadataEntity] ) -> None: # The family belongs to `data_type`; a codec that happens to be named # `r8` must not reach it. @@ -80,14 +79,14 @@ def test_r_shaped_names_resolve_to_nothing_outside_data_types( _UNCLAIMED = st.text(min_size=1).filter( lambda name: ( not (name.startswith("r") and name[1:].isdigit()) - and name not in CORE_AND_EXTENSIONS.entities["data_type"] + and name not in CORE_AND_EXTENSIONS.tables[DataTypeEntity] ) ) @given(name=_UNCLAIMED) def test_a_name_no_entity_claims_resolves_to_nothing(name: str) -> None: - assert CORE_AND_EXTENSIONS.resolve(DATA_TYPE, name) is None + assert CORE_AND_EXTENSIONS.resolve(DataTypeEntity, name) is None def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: From 342194085997a9f0b5a52587d9cf51f5589a6285 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 15:53:22 +0200 Subject: [PATCH 082/107] refactor(zarr-metadata): to_json is abstract; each entity writes a literal of its own JSON type Review comment: a base `to_json` that builds a `dict[str, object]` and casts it to a type parameter it cannot know is wrong. It is abstract now. Each of the 33 entities writes its own, as a literal of the TypedDict it names -- the bare name when every member is absent, the object otherwise, a contained entity through `written` -- and pyright holds the literal to the TypedDict: a key it does not declare, a required one left out, a value of the wrong type is a static error. Gone with the cast: the class-creation check that compared the named type with the fields and its helpers, `json_type_of`, the generic `configuration()` and the rendering walk it needed, the `must_understand` class variable nothing read any more, and the test oracle that compiled the named type -- the examples table now round-trips each document through `coerce` and `to_json` instead. `ArrayDocumentV3.to_json` writes its five fields by name. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 19 +- .../src/zarr_metadata/v3/_document.py | 22 +- .../src/zarr_metadata/v3/_entity.py | 297 ++---------------- .../src/zarr_metadata/v3/_registry.py | 5 +- .../v3/chunk_grid/rectilinear.py | 6 + .../zarr_metadata/v3/chunk_grid/regular.py | 3 + .../v3/chunk_key_encoding/default.py | 5 + .../zarr_metadata/v3/chunk_key_encoding/v2.py | 5 + .../src/zarr_metadata/v3/codec/blosc.py | 11 + .../src/zarr_metadata/v3/codec/bytes.py | 5 + .../src/zarr_metadata/v3/codec/cast_value.py | 14 + .../src/zarr_metadata/v3/codec/crc32c.py | 3 + .../src/zarr_metadata/v3/codec/gzip.py | 3 + .../zarr_metadata/v3/codec/scale_offset.py | 13 + .../v3/codec/sharding_indexed.py | 11 + .../src/zarr_metadata/v3/codec/transpose.py | 3 + .../src/zarr_metadata/v3/codec/zstd.py | 6 + .../src/zarr_metadata/v3/data_type/bool.py | 3 + .../src/zarr_metadata/v3/data_type/bytes.py | 3 + .../zarr_metadata/v3/data_type/complex128.py | 3 + .../zarr_metadata/v3/data_type/complex64.py | 3 + .../src/zarr_metadata/v3/data_type/float16.py | 3 + .../src/zarr_metadata/v3/data_type/float32.py | 3 + .../src/zarr_metadata/v3/data_type/float64.py | 3 + .../src/zarr_metadata/v3/data_type/int16.py | 3 + .../src/zarr_metadata/v3/data_type/int32.py | 3 + .../src/zarr_metadata/v3/data_type/int64.py | 3 + .../src/zarr_metadata/v3/data_type/int8.py | 3 + .../v3/data_type/numpy_datetime64.py | 6 + .../v3/data_type/numpy_timedelta64.py | 6 + .../src/zarr_metadata/v3/data_type/string.py | 3 + .../src/zarr_metadata/v3/data_type/struct.py | 11 + .../src/zarr_metadata/v3/data_type/uint16.py | 3 + .../src/zarr_metadata/v3/data_type/uint32.py | 3 + .../src/zarr_metadata/v3/data_type/uint64.py | 3 + .../src/zarr_metadata/v3/data_type/uint8.py | 3 + .../src/zarr_metadata/v3/entity.py | 39 ++- .../tests/v3/test_acme_affine.py | 9 + .../tests/v3/test_acme_decimal.py | 6 + .../zarr-metadata/tests/v3/test_entities.py | 53 ++-- .../tests/v3/test_extension_api.py | 105 +++---- 41 files changed, 331 insertions(+), 383 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 356d8d2dd5..9cfb9d2bce 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -23,14 +23,9 @@ documents report identically. An entity names its own JSON type as the base's argument -- `class GzipCodec(CodecEntity[GzipCodecMetadata])` -- and `to_json` returns -it. That is the one place the correspondence between an entity and its -public JSON type is written, and it is held to twice: class creation -refuses a named type whose shape (bare name, object, or either) disagrees -with what the members make the entity write, and a property test draws -documents from the named type, reads them in, writes them back, and judges -the result against the named type with the package's own compiler. The -parameter has a default, so a bare `CodecEntity` -- in a field, a table, a -scope -- admits every codec, and a third party may leave it unnamed. +it. The parameter has a default, so a bare `CodecEntity` -- in a field, a +table, a scope -- admits every codec, and a third party may leave it +unnamed. `crc32c` names `Crc32cCodecName` alone: it has no members, so it only ever writes the bare name, and the union is what a document may spell, not what the entity writes. For the conformance test to use the @@ -88,6 +83,14 @@ registration checks for that are gone. Which of a document's fields holds which kind is the document's own knowledge, in one place; the entity layer no longer names a document field. +`to_json` is abstract: each entity writes its own, as a literal of its +JSON type, and the type checker holds the literal to the TypedDict -- +which is what a base method building a `dict[str, object]` and casting +it to the named type could only assert. The class-creation check that +compared the named type with the fields, the generic `configuration()` +and the rendering walk it needed are gone with the cast; `written` +renders a contained field. + One thing this does not change, under mypy. An entity's JSON type is a TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as `Mapping[str, object]`, never as the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 0f1ed2d6a6..978ab9cda9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -40,8 +40,8 @@ StorageTransformerEntity, canonicalize_nested, contains_entity, - render_nested, within, + written, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context @@ -116,16 +116,20 @@ def to_json(self) -> dict[str, object]: Faithful to what was read, member for member; the fields that are not extension points come back exactly as the document had - them. Ask `canonical` first for the simplest equivalent spelling. + them, and a field the document did not have is not invented. + Ask `canonical` first for the simplest equivalent spelling. """ - # Only the fields the document wrote: an absent one was read as an - # `Opaque` standing in, and writing it back would invent a null. - rendered = { - name: render_nested(annotation, getattr(self, name)) - for name, annotation in field_hints(type(self)).items() - if contains_entity(annotation) and name in self.document + rendered: dict[str, object] = { + "data_type": written(self.data_type), + "chunk_grid": written(self.chunk_grid), + "chunk_key_encoding": written(self.chunk_key_encoding), + "codecs": tuple(written(codec) for codec in self.codecs), + "storage_transformers": tuple(written(entry) for entry in self.storage_transformers), + } + return { + **self.document, + **{key: value for key, value in rendered.items() if key in self.document}, } - return {**self.document, **rendered} @classmethod def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> ArrayDocumentV3: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 6f1d753814..be116184ae 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -34,7 +34,6 @@ # creation, and a name that exists only for the type checker is a NameError # then -- for this package and for any tool introspecting an entity. from collections.abc import Callable, Mapping # noqa: TC003 -from copy import deepcopy from dataclasses import MISSING, Field, dataclass, is_dataclass, replace from typing import ( TYPE_CHECKING, @@ -42,7 +41,6 @@ Final, Generic, Literal, - NotRequired, TypeAlias, cast, final, @@ -173,24 +171,6 @@ def _is_entity_kind(candidate: object) -> TypeIs[type[MetadataEntity]]: return isinstance(candidate, type) and issubclass(candidate, MetadataEntity) -def json_type_of(cls: type[MetadataEntity]) -> object: - """The JSON type `cls` names for `to_json`; the default if it names none. - - Read off the subscripted base the class -- or the nearest ancestor - that did -- was declared with, `CodecEntity[GzipCodecMetadata]`: the - one place the correspondence between an entity and its public JSON - type is written. - """ - for klass in cls.__mro__: - for base in klass.__dict__.get("__orig_bases__", ()): - origin = get_origin(base) - if isinstance(origin, type) and issubclass(origin, MetadataEntity): - arguments = get_args(base) - if len(arguments) == 1 and not isinstance(arguments[0], TypeVar): - return arguments[0] - return ZarrV3MetadataFieldJSON - - def contains_entity(annotation: object) -> bool: """Whether a value of this type holds a nested metadata field anywhere in it.""" inner, _ = strip_annotation(annotation) @@ -291,31 +271,17 @@ def _resolve( return value, () -def render_nested(annotation: object, value: object) -> object: - """`value` as a document would write it: every nested entity in its JSON form.""" - if is_entity(value): +def written(value: MetadataEntity | Opaque) -> ZarrV3MetadataFieldJSON: + """A contained metadata field as a document would write it. + + The entity's own JSON, or the JSON an `Opaque` kept. An `Opaque` + inside a built entity is out of scope -- an inner name no entity in + scope claimed -- and its JSON passed the envelope check as a metadata + field, which is what the cast says. + """ + if isinstance(value, MetadataEntity): return value.to_json() - if isinstance(value, Opaque): - return value.json - inner, _ = strip_annotation(annotation) - if is_union(inner): - branch = _fitting_branch(inner, value) - return value if branch is None else render_nested(branch, value) - if get_origin(inner) is tuple: - entries = cast("tuple[object, ...]", value) - return tuple( - render_nested(element, entry) - for element, entry in zip( - element_annotations(inner, len(entries)), entries, strict=True - ) - ) - if isinstance(inner, type) and is_dataclass(inner) and not is_metadata_field_type(inner): - return { - name: render_nested(field_annotation, getattr(value, name)) - for name, field_annotation in field_hints(inner).items() - if getattr(value, name) is not UNSET - } - return value + return cast("ZarrV3MetadataFieldJSON", value.json) def canonicalize_nested(annotation: object, value: object) -> object: @@ -530,126 +496,6 @@ def _owed_class_variables_are_declared(cls: type[MetadataEntity]) -> str | None: ) -def _is_name_type(part: object) -> bool: - """A JSON type for the bare-name spelling: `str`, a `Literal` of names, or a `NewType` of `str`.""" - return ( - part is str - or (get_origin(part) is Literal and all(isinstance(v, str) for v in get_args(part))) - or getattr(part, "__supertype__", None) is str - ) - - -def _unaccepted(cls: type[MetadataEntity], name_type: object) -> list[str]: - """The names a `Literal` name type lists that the entity does not accept.""" - if get_origin(name_type) is not Literal: - return [] - return [value for value in get_args(name_type) if not cls.accepts(value)] - - -def _named_json_type_matches_what_is_written(cls: type[MetadataEntity]) -> str | None: - # The named type is a promise about what `to_json` writes, held to - # key by key: the spellings it admits are the ones the members make - # the entity write, its names are ones the entity accepts, and its - # configuration keys are the members. Value types are not compared - # -- a nested entity's JSON type and its field type are different - # spellings of one thing -- so that much stays with the tests. - json_type = json_type_of(cls) - if json_type is ZarrV3MetadataFieldJSON: - return None - parts = get_args(json_type) if is_union(json_type) else (json_type,) - objects = [part for part in parts if is_typeddict(part)] - names = [part for part in parts if _is_name_type(part)] - if len(objects) > 1 or len(names) > 1 or len(objects) + len(names) != len(parts): - return ( - f"{cls.__name__} names {json_type!r} as its JSON type, which is not an object " - "TypedDict, a name type, or a union of one of each" - ) - members = _members(cls) - writes_bare = not any(members.values()) and cls.must_understand - writes_object = len(members) != 0 or not cls.must_understand - found: list[str] = [] - if writes_bare and len(names) == 0: - found.append("lacks the bare name the entity writes when every member is absent") - if not writes_bare and len(names) != 0: - found.append("admits a bare name, which the entity never writes") - if writes_object and len(objects) == 0: - found.append("lacks the object the entity writes") - if not writes_object and len(objects) != 0: - found.append("admits an object, which the entity never writes") - found.extend( - f"lists the name(s) {', '.join(map(repr, unaccepted))}, which the entity does not accept" - for name_type in names - if len(unaccepted := _unaccepted(cls, name_type)) != 0 - ) - for obj in objects: - try: - resolved = get_type_hints(obj, include_extras=True) - except NameError: - # Declared inside a function under postponed annotations: the - # names it uses are not reachable, so its keys go unjudged. - continue - hints = {key: strip_annotation(value)[0] for key, value in resolved.items()} - # Requiredness from the resolved hints, not `__required_keys__`: - # under postponed annotations a TypedDict's own metaclass cannot - # see `NotRequired` inside a string. - required = {key for key, value in resolved.items() if get_origin(value) is not NotRequired} - extra = sorted(hints.keys() - {"name", "configuration", "must_understand"}) - if len(extra) != 0: - found.append(f"has the key(s) {', '.join(extra)}, which no envelope has") - if "name" not in hints: - found.append("has no name key") - elif len(unaccepted := _unaccepted(cls, hints["name"])) != 0: - found.append( - f"names {', '.join(map(repr, unaccepted))}, which the entity does not accept" - ) - if not cls.must_understand and "must_understand" not in hints: - found.append("has no must_understand key, which the entity writes") - if len(members) == 0: - if "configuration" in hints: - found.append("has a configuration key, and the entity has no members") - continue - if "configuration" not in hints: - found.append("has no configuration key, and the entity has members") - continue - if ("configuration" in required) != any(members.values()): - found.append( - "has configuration " - + ("required" if "configuration" in required else "optional") - + ", but a member is " - + ("required" if any(members.values()) else "not required") - ) - configuration = hints["configuration"] - if not is_typeddict(configuration): - continue - try: - configuration_hints = get_type_hints(configuration, include_extras=True) - except NameError: - continue - keys = configuration_hints.keys() - if set(keys) != set(members): - found.append( - f"has configuration keys {sorted(keys)!r} where the members are {sorted(members)!r}" - ) - continue - required_keys = { - key - for key, value in configuration_hints.items() - if get_origin(value) is not NotRequired - } - misstated = sorted( - key - for key, member_required in members.items() - if (key in required_keys) != member_required - ) - if len(misstated) != 0: - found.append( - f"states {', '.join(misstated)} with a requiredness the field does not give it" - ) - if len(found) == 0: - return None - return f"{cls.__name__} names {json_type!r} as its JSON type, which " + "; ".join(found) - - def _codecs_are_of_a_kind(cls: type[MetadataEntity]) -> str | None: # The kind is the base class, and what a kind must answer is abstract # on it; a codec that skips the kind classes skips that. @@ -743,7 +589,6 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: _nested_fields_admit_opaque, _fields_do_not_shadow_class_variables, _owed_class_variables_are_declared, - _named_json_type_matches_what_is_written, _codecs_are_of_a_kind, _class_variables_hold_listed_values, _optional_members_default_to_unset, @@ -775,27 +620,9 @@ class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): say beyond their types, a `__post_init__` that collects every problem and raises `MetadataValidationError` once -- so `BloscCodec(clevel=99)` raises, and `coerce` reports the same problems instead. `coerce`, - `configuration`, `to_json` and `canonical` are written once here - against what the fields say, read off them as needed. - """ - - must_understand: ClassVar[bool] = True - """Whether a reader must understand this entity to read the array. - - `True`, the default and every codec: a reader that does not know it - may not skip it. - - A property of the *kind* of metadata, not of a use of it: a codec is - something you must understand, every time it appears, because - ignoring one gives wrong bytes. Consolidated metadata is the opposite - and is unconditionally skippable. Neither is a per-occurrence choice, - so neither is a configuration member -- which is why this is a class - variable and not a field. - - The spec permits `must_understand: false` on a codec; this package - treats that as an oversight and refuses it. Where the flag does earn - its keep -- an unknown top-level extension field a reader really can - skip -- it stays per-occurrence, on `ZarrV3NamedConfig`. + `coerce` and `canonical` are written once here against what the + fields say, read off them as needed; `to_json` is the entity's own, + a literal of its JSON type. """ identifier: ClassVar[str] @@ -951,6 +778,25 @@ def canonical(self) -> Self: ) return walked.simplified() + @abstractmethod + def to_json(self) -> JSONT_co: + """This entity as a document would write it: a literal of its own JSON type. + + Faithful to every member it holds: read a document, write it + back, and those come out as they went in. Ask `canonical` first + if you want the simplest equivalent spelling. The envelope's + spelling is the one thing not preserved, because the entity does + not model it: a bare name, `{"name": x}` and `{"name": x, + "configuration": {}}` all read to the same entity, and the entity + writes the bare name when every member it holds is absent. + + Written per entity, as a literal of the TypedDict named as the + base's argument -- `CodecEntity[GzipCodecObject]` -- which is + what holds it to that type: pyright checks the literal's keys and + values against the TypedDict. A contained entity is written with + `written`. + """ + def simplified(self) -> Self: """This entity with its own members in their simplest equivalent spelling. @@ -964,83 +810,6 @@ def simplified(self) -> Self: """ return self - def configuration(self) -> dict[str, object]: - """This entity's configuration, as the document would write it. - - Faithful to every member the entity holds: `to_json` is - serialization, not canonicalization, so nothing is simplified - here. A contained entity is rendered as its own `to_json`, by - walking the fields that hold one. - - Absent optional members are left out, which is what makes the - bare-name spelling reachable. Absence is `UNSET`, never `None`: - this package holds `None` to mean a JSON `null` the document - actually wrote, and `scale_offset` is a real case where `null` - and absent are different documents. - - Deep-copied, because a member can be an arbitrary JSON value: a - `scale_offset` offset may be an object, and handing the caller - the entity's own dict would let them mutate a frozen entity - through the document it returned. - """ - members = self._configuration_members() - for name, annotation in _nested(type(self)).items(): - if name in members: - members[name] = render_nested(annotation, members[name]) - return deepcopy(members) - - def _configuration_members(self) -> dict[str, object]: - """The members a configuration object would spell out. - - Every field but one the envelope carries some other way -- the - `r` width, which lives in the name. - """ - return { - key: value for key in _members(type(self)) if (value := getattr(self, key)) is not UNSET - } - - def to_json(self) -> JSONT_co: - """This entity as a document would write it. - - Faithful to every member it models: read a document, write it - back, and those come out as they went in. Ask `canonical` first - if you want the simplest equivalent spelling. - - A member this entity does not model is not one of them. It is - reported as `unknown_key` and not held, so writing back drops it - -- which only a caller who took the problems as data and went on - past that one can reach, because `from_json` raises on it. A - caller who needs the bytes preserved has the JSON it passed in, - and `Opaque` is where unmodelled metadata belongs. - - What is *not* preserved is the envelope's spelling, because the - entity does not model it: a bare name, `{"name": x}`, and - `{"name": x, "configuration": {}}` all mean the same and all read - to the same entity, so all three write back as the bare name. - `must_understand` follows the entity's own class variable, so it - is omitted for everything this package models today. - - The return type is the entity's own JSON type, named as the - base's argument -- `GzipCodecObject`, `BytesCodecObject | - BytesCodecName` -- and the `cast` below is the one place the - package asserts that the dict it builds has that shape. Asserted - rather than proven because a TypedDict cannot be built member by - member from `dict[str, object]`; held to, twice: `__init_subclass__` - refuses a named type whose shape disagrees with whether this - entity ever writes a bare name or an object, and - `tests/v3/test_entities.py` compiles the named type with - `check_for` and runs every entity's output through it. - """ - configuration = self.configuration() - if len(configuration) == 0 and type(self).must_understand: - return cast("JSONT_co", type(self).identifier) - entry: dict[str, object] = {"name": type(self).identifier} - if len(configuration) != 0: - entry["configuration"] = configuration - if not type(self).must_understand: - entry["must_understand"] = False - return cast("JSONT_co", entry) - @dataclass(frozen=True) class CodecEntity(MetadataEntity[JSONT_co], base=True): @@ -1205,11 +974,11 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "is_json_value", "is_metadata_field", "is_str", - "json_type_of", "kind_of", "named_configuration", "one_of", "problem", "sequence_of", "within", + "written", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 8b1c3e2fe8..2caf1226a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -228,10 +228,7 @@ def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: raise TypeError(msg) if inspect.isabstract(entity): left = ", ".join(sorted(entity.__abstractmethods__)) - msg = ( - f"{entity.__name__} does not define {left}, which its base leaves abstract; " - "define it, if only to return the same thing as `incoming` or ()" - ) + msg = f"{entity.__name__} does not define {left}, which its base leaves abstract" raise TypeError(msg) if "__dataclass_fields__" not in vars(entity) and any( not is_class_var(annotation) for annotation in own_annotations(entity).values() diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 952199a541..f24053e6db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -242,3 +242,9 @@ def simplified(self) -> Self: grid, and the encoded one stays the same size as the array grows. """ return replace(self, chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) + + def to_json(self) -> RectilinearChunkGridObject: + return { + "name": "rectilinear", + "configuration": {"kind": self.kind, "chunk_shapes": self.chunk_shapes}, + } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index dc791ef697..951131ffb5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -99,3 +99,6 @@ def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: def grid(self, array_shape: object) -> ChunkGrid: """One extent per axis, the same for every chunk on that axis.""" return ChunkGrid.regular(self.chunk_shape) + + def to_json(self) -> RegularChunkGridObject: + return {"name": "regular", "configuration": {"chunk_shape": self.chunk_shape}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index d486864da3..6d89dc9dae 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -77,3 +77,8 @@ class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity[DefaultChunkKeyEncodingMeta separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME + + def to_json(self) -> DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName: + if self.separator is UNSET: + return "default" + return {"name": "default", "configuration": {"separator": self.separator}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 87cac6d1ae..dd8fcbd157 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -83,3 +83,8 @@ class V2ChunkKeyEncoding(ChunkKeyEncodingEntity[V2ChunkKeyEncodingMetadata]): separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME + + def to_json(self) -> V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName: + if self.separator is UNSET: + return "v2" + return {"name": "v2", "configuration": {"separator": self.separator}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 15a2be84ee..ef6173d5e1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -161,3 +161,14 @@ def simplified(self) -> Self: if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: return self return replace(self, typesize=UNSET) + + def to_json(self) -> BloscCodecObject: + configuration: BloscCodecConfiguration = { + "cname": self.cname, + "clevel": self.clevel, + "shuffle": self.shuffle, + "blocksize": self.blocksize, + } + if self.typesize is not UNSET: + configuration["typesize"] = self.typesize + return {"name": "blosc", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 9928a9f6f5..01f8eabec8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -117,3 +117,8 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb "missing_key", ) return () + + def to_json(self) -> BytesCodecObject | BytesCodecName: + if self.endian is UNSET: + return "bytes" + return {"name": "bytes", "configuration": {"endian": self.endian}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index fa60991fb2..76af074139 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -4,6 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ +from copy import deepcopy from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired @@ -16,6 +17,7 @@ ArrayArrayCodec, DataTypeEntity, Opaque, + written, ) from zarr_metadata.v3._parts import ArrayParts @@ -143,3 +145,15 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type return incoming.with_data_type(data_type if isinstance(data_type, DataTypeEntity) else None) + + def to_json(self) -> CastValueCodecObject: + configuration: CastValueCodecConfiguration = {"data_type": written(self.data_type)} + if self.rounding is not UNSET: + configuration["rounding"] = self.rounding + if self.out_of_range is not UNSET: + configuration["out_of_range"] = self.out_of_range + if self.scalar_map is not UNSET: + # Copied: the document handed out must not be a handle on + # this frozen entity's own map. + configuration["scalar_map"] = deepcopy(self.scalar_map) + return {"name": "cast_value", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 6bfce5e188..d5f8635464 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -67,3 +67,6 @@ class Crc32cCodec(BytesBytesCodec[Crc32cCodecName]): """ identifier: ClassVar[str] = CRC32C_CODEC_NAME + + def to_json(self) -> Crc32cCodecName: + return "crc32c" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index a90ffecf29..cdeff9da92 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -81,3 +81,6 @@ def __post_init__(self) -> None: ("level",), f"expected an integer in [0, 9], got {self.level}", "invalid_value" ) ) + + def to_json(self) -> GzipCodecObject: + return {"name": "gzip", "configuration": {"level": self.level}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index b3295fd458..f2dd6967e9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -4,6 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ +from copy import deepcopy from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired @@ -109,3 +110,15 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: longer changes the element type -- only the values. """ return incoming + + def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: + configuration: ScaleOffsetCodecConfiguration = {} + # Copied: a member may be a JSON object, and the document handed + # out must not be a handle on this frozen entity. + if self.offset is not UNSET: + configuration["offset"] = deepcopy(self.offset) + if self.scale is not UNSET: + configuration["scale"] = deepcopy(self.scale) + if len(configuration) == 0: + return "scale_offset" + return {"name": "scale_offset", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index d59165bdc0..ea2c5295e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -18,6 +18,7 @@ CodecEntity, Opaque, problem, + written, ) from zarr_metadata.v3._parts import ( UNKNOWN_GRID, @@ -196,3 +197,13 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation ) ) return tuple(found) + + def to_json(self) -> ShardingIndexedCodecObject: + configuration: ShardingIndexedCodecConfiguration = { + "chunk_shape": self.chunk_shape, + "codecs": tuple(written(codec) for codec in self.codecs), + "index_codecs": tuple(written(codec) for codec in self.index_codecs), + } + if self.index_location is not UNSET: + configuration["index_location"] = self.index_location + return {"name": "sharding_indexed", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index db74338e61..9a0b095854 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -108,3 +108,6 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: longer the grid the document wrote. """ return incoming.with_grid(incoming.grid.permuted(self.order)) + + def to_json(self) -> TransposeCodecObject: + return {"name": "transpose", "configuration": {"order": self.order}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index e8dde0501c..f62de7e36f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -92,3 +92,9 @@ def __post_init__(self) -> None: "invalid_value", ) ) + + def to_json(self) -> ZstdCodecObject: + configuration: ZstdCodecConfiguration = {"level": self.level} + if self.checksum is not UNSET: + configuration["checksum"] = self.checksum + return {"name": "zstd", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index 3c49b4349d..4eb4d52f15 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -44,3 +44,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP if not isinstance(value, bool): return problem(loc, f"expected a boolean, got {value!r}", "invalid_value") return () + + def to_json(self) -> BoolDataTypeName: + return "bool" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index f2d2fa38fe..3c759ed93d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -77,3 +77,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP ) return () return byte_values(value, None, loc) + + def to_json(self) -> BytesDataTypeName: + return "bytes" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index eb5d554edb..951a6b7850 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -48,3 +48,6 @@ class Complex128DataType(ComplexDataType[Complex128DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" component: ClassVar[type[FloatDataType]] = Float64DataType identifier: ClassVar[str] = COMPLEX128_DATA_TYPE_NAME + + def to_json(self) -> Complex128DataTypeName: + return "complex128" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 715557a901..fcf2995896 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -48,3 +48,6 @@ class Complex64DataType(ComplexDataType[Complex64DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" component: ClassVar[type[FloatDataType]] = Float32DataType identifier: ClassVar[str] = COMPLEX64_DATA_TYPE_NAME + + def to_json(self) -> Complex64DataTypeName: + return "complex64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 10cb15ce14..68c9e26ad8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -88,3 +88,6 @@ class Float16DataType(FloatDataType[Float16DataTypeName]): hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float16) largest: ClassVar[float | None] = 65504.0 identifier: ClassVar[str] = FLOAT16_DATA_TYPE_NAME + + def to_json(self) -> Float16DataTypeName: + return "float16" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 46658f93a9..f834025dcf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -88,3 +88,6 @@ class Float32DataType(FloatDataType[Float32DataTypeName]): hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float32) largest: ClassVar[float | None] = 3.4028235e38 identifier: ClassVar[str] = FLOAT32_DATA_TYPE_NAME + + def to_json(self) -> Float32DataTypeName: + return "float32" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 531407fb07..06ad561df5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -89,3 +89,6 @@ class Float64DataType(FloatDataType[Float64DataTypeName]): hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float64) largest: ClassVar[float | None] = None identifier: ClassVar[str] = FLOAT64_DATA_TYPE_NAME + + def to_json(self) -> Float64DataTypeName: + return "float64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index d4536b2815..400104a997 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -35,3 +35,6 @@ class Int16DataType(IntegerDataType[Int16DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (-32768, 32767) identifier: ClassVar[str] = INT16_DATA_TYPE_NAME + + def to_json(self) -> Int16DataTypeName: + return "int16" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 0d57f988c5..25d78badb8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -35,3 +35,6 @@ class Int32DataType(IntegerDataType[Int32DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (-2147483648, 2147483647) identifier: ClassVar[str] = INT32_DATA_TYPE_NAME + + def to_json(self) -> Int32DataTypeName: + return "int32" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index b957738f45..9894461834 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -35,3 +35,6 @@ class Int64DataType(IntegerDataType[Int64DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (-9223372036854775808, 9223372036854775807) identifier: ClassVar[str] = INT64_DATA_TYPE_NAME + + def to_json(self) -> Int64DataTypeName: + return "int64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index bfecdacb04..fc9feb0879 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -35,3 +35,6 @@ class Int8DataType(IntegerDataType[Int8DataTypeName]): scalar_storage: ClassVar[StorageClass] = "single_byte" bounds: ClassVar[tuple[int, int]] = (-128, 127) identifier: ClassVar[str] = INT8_DATA_TYPE_NAME + + def to_json(self) -> Int8DataTypeName: + return "int8" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index e2d4003bd7..63d812c360 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -89,3 +89,9 @@ def __post_init__(self) -> None: "invalid_value", ) ) + + def to_json(self) -> NumpyDatetime64: + return { + "name": "numpy.datetime64", + "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, + } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 03fc1d72e3..cc4105ee4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -92,3 +92,9 @@ def __post_init__(self) -> None: "invalid_value", ) ) + + def to_json(self) -> NumpyTimedelta64: + return { + "name": "numpy.timedelta64", + "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, + } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 5000788e64..9f9b498141 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -44,3 +44,6 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP if not isinstance(value, str): return problem(loc, f"expected a string, got {value!r}", "invalid_value") return () + + def to_json(self) -> StringDataTypeName: + return "string" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 2b7b5659dc..308c9fed88 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -19,6 +19,7 @@ Opaque, StorageClass, problem, + written, ) STRUCT_DATA_TYPE_NAME: Final = "struct" @@ -95,6 +96,10 @@ class StructFieldComponent: data_type: DataTypeEntity | Opaque +def _written_field(field: StructFieldComponent) -> StructField: + return {"name": field.name, "data_type": written(field.data_type)} + + @dataclass(frozen=True) class StructDataType(DataTypeEntity[Struct]): """The `struct` data type, coerced from its metadata. @@ -209,3 +214,9 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP for key in sorted(fills.keys() - declared) ) return tuple(found) + + def to_json(self) -> Struct: + return { + "name": "struct", + "configuration": {"fields": tuple(_written_field(field) for field in self.fields)}, + } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 19055a172a..0bbaa52bc9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -35,3 +35,6 @@ class Uint16DataType(IntegerDataType[Uint16DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (0, 65535) identifier: ClassVar[str] = UINT16_DATA_TYPE_NAME + + def to_json(self) -> Uint16DataTypeName: + return "uint16" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index d07bc8ce32..fa0189ba0d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -35,3 +35,6 @@ class Uint32DataType(IntegerDataType[Uint32DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (0, 4294967295) identifier: ClassVar[str] = UINT32_DATA_TYPE_NAME + + def to_json(self) -> Uint32DataTypeName: + return "uint32" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 89b2816b3b..57a6446ff9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -35,3 +35,6 @@ class Uint64DataType(IntegerDataType[Uint64DataTypeName]): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (0, 18446744073709551615) identifier: ClassVar[str] = UINT64_DATA_TYPE_NAME + + def to_json(self) -> Uint64DataTypeName: + return "uint64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index aa9b8a04d9..7b41b54144 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -35,3 +35,6 @@ class Uint8DataType(IntegerDataType[Uint8DataTypeName]): scalar_storage: ClassVar[StorageClass] = "single_byte" bounds: ClassVar[tuple[int, int]] = (0, 255) identifier: ClassVar[str] = UINT8_DATA_TYPE_NAME + + def to_json(self) -> Uint8DataTypeName: + return "uint8" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index b80d964f39..244c8e68d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -44,7 +44,9 @@ class to a scope. Complete, and runnable as written: from dataclasses import dataclass - from typing import ClassVar + from typing import ClassVar, Literal, NotRequired + + from typing_extensions import TypedDict from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3.entity import ( @@ -55,8 +57,15 @@ class to a scope. Complete, and runnable as written: problem, ) + class AcmeLz4Configuration(TypedDict, closed=True): + acceleration: NotRequired[int] + + class AcmeLz4Object(TypedDict, closed=True): + name: Literal["acme.lz4"] + configuration: AcmeLz4Configuration + @dataclass(frozen=True) # load-bearing: `coerce` builds the entity with cls(**members) - class AcmeLz4Codec(BytesBytesCodec): + class AcmeLz4Codec(BytesBytesCodec[AcmeLz4Object | Literal["acme.lz4"]]): acceleration: int | UNSET = UNSET # optional: defaults to UNSET, never to a value identifier: ClassVar[str] = "acme.lz4" @@ -71,6 +80,11 @@ def __post_init__(self) -> None: ) ) + def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: + if self.acceleration is UNSET: + return "acme.lz4" + return {"name": "acme.lz4", "configuration": {"acceleration": self.acceleration}} + SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec) validate_array_metadata_v3(document, context=SCOPE) @@ -98,8 +112,11 @@ def __post_init__(self) -> None: read: a member of the wrong type is reported and the entity is not built. **What an entity answers for itself**, beyond its fields. `to_json`, -`canonical` and `coerce` are written once in the base; an entity whose -own members have two spellings that mean the same overrides +abstract: the entity as a document writes it, as a literal of its own +TypedDict, which pyright holds to that type -- the bare name when every +member is absent, the object otherwise, a contained entity through +`written`. `coerce` and `canonical` are written once in the base; an +entity whose own members have two spellings that mean the same overrides `simplified`, which `canonical` calls (overriding `canonical` itself is refused). Then, by kind: @@ -134,13 +151,11 @@ class creation or registration with a message that says what to write. key is its `identifier`, so `extended_with` takes the classes and nothing can be misfiled. -**Naming the JSON type.** `CodecEntity[AcmeLz4Metadata]` types `to_json` -as your own TypedDict rather than as any metadata field. The shape is -`{name: Literal["acme.lz4"], configuration: AcmeLz4Configuration, -must_understand: NotRequired[bool]}`, in a union with the name literal -only if no member is required; class creation holds it to the entity -key by key (names it accepts, the members as configuration keys with -the members' requiredness), and the tests hold the value types. +**Naming the JSON type.** `BytesBytesCodec[AcmeLz4Object | Literal["acme.lz4"]]` +types `to_json` as your own JSON type rather than as any metadata field, +and pyright checks the literal `to_json` returns against it: a key it +does not declare, a required one left out, a value of the wrong type is +a static error. Left unnamed, `to_json` is typed as any metadata field. Two complete extensions written against this module alone, as tests: `tests/v3/test_acme_affine.py` (an `array_array` codec with a number, an @@ -195,6 +210,7 @@ class creation or registration with a message that says what to write. named_configuration, problem, within, + written, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid, Extents from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context @@ -243,4 +259,5 @@ class creation or registration with a message that says what to write. "named_configuration", "problem", "within", + "written", ] diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 4f49ffc41f..d6991d8ecd 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -32,6 +32,7 @@ ValidationProblem, ZarrV3MetadataFieldJSON, problem, + written, ) @@ -79,6 +80,14 @@ def simplified(self) -> Self: """An offset of 0 is the identity, and absent says the same.""" return replace(self, offset=UNSET) if self.offset == 0 else self + def to_json(self) -> AcmeAffineObject: + configuration: AcmeAffineConfiguration = {"scale": self.scale} + if self.offset is not UNSET: + configuration["offset"] = self.offset + if self.dtype is not UNSET: + configuration["dtype"] = written(self.dtype) + return {"name": "acme.affine", "configuration": configuration} + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: data_type = incoming.data_type if incoming is not None else None if data_type is None or data_type.storage_class() != "variable_length": diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 3a943d0e98..6a2c79c5b3 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -106,6 +106,12 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) + def to_json(self) -> AcmeDecimal: + return { + "name": "acme.decimal", + "configuration": {"precision": self.precision, "scale": self.scale}, + } + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """A decimal literal whose digits fit `precision` and `scale`. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index b8264cebcf..ff77d45b5b 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -12,6 +12,7 @@ import copy import dataclasses from typing import ( + TYPE_CHECKING, Any, ClassVar, Self, @@ -23,12 +24,13 @@ from tests.helpers import configuration_of from tests.rules.strategies import valid_documents + +if TYPE_CHECKING: + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -from zarr_metadata.v3._compile import check_for from zarr_metadata.v3._document import read_array_v3 -from zarr_metadata.v3._entity import json_type_of from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, @@ -134,13 +136,6 @@ } -@pytest.mark.parametrize("entity", ENTITIES.values(), ids=list(ENTITIES)) -def test_every_entity_names_its_json_type(entity: type[MetadataEntity]) -> None: - # The default is not wrong, only uninformative; every entity this - # package models says exactly what it writes. - assert json_type_of(entity) is not ZarrV3MetadataFieldJSON - - # One or more documents each entity reads, spelled to reach both shapes # where the entity has both: the bare name when every member is absent, # the object otherwise. `st.from_type` over the named types cannot serve @@ -247,15 +242,14 @@ def test_every_entity_names_its_json_type(entity: type[MetadataEntity]) -> None: } -def _assert_conforms(entity: MetadataEntity) -> None: - # The one `cast` in `to_json` asserts that what it builds has the - # entity's named type. This is that assertion, checked: the named - # type compiled by the package's own compiler, and the output run - # through it. - json_type = json_type_of(type(entity)) - check = check_for(json_type) - assert check is not None, f"{json_type!r} is not a shape the compiler reads" - assert check(entity.to_json(), ()) == () +def _round_trips(entity: type[MetadataEntity], document: object) -> MetadataEntity: + read, problems = entity.coerce(document, CORE_AND_EXTENSIONS) + assert problems == () + assert read is not None + again, problems = entity.coerce(read.to_json(), CORE_AND_EXTENSIONS) + assert problems == () + assert again == read + return read @pytest.mark.parametrize( @@ -265,26 +259,22 @@ def _assert_conforms(entity: MetadataEntity) -> None: f"{key}:{index}" for key, documents in EXAMPLES.items() for index in range(len(documents)) ], ) -def test_to_json_conforms_to_the_named_json_type( +def test_to_json_reads_back_to_the_same_entity( entity: type[MetadataEntity], document: object ) -> None: - read, problems = entity.coerce(document, CORE_AND_EXTENSIONS) - assert problems == () - assert read is not None - _assert_conforms(read) + # What `to_json` writes, `coerce` reads to the entity that wrote it: + # every member is written, in the spelling the reader expects. + _round_trips(entity, document) @given(document=valid_documents()) @settings(max_examples=50, deadline=None) -def test_to_json_conforms_across_a_valid_document(document: dict[str, object]) -> None: - # The top-level entities of documents valid by construction, for the - # variation the examples fix: permutations, chunk shapes, an index - # pipeline. A nested entity is some entity's top-level example. +def test_to_json_reads_back_across_a_valid_document(document: dict[str, object]) -> None: array, problems = read_array_v3(document, CORE_AND_EXTENSIONS) assert problems == () for entity in (array.data_type, array.chunk_grid, array.chunk_key_encoding, *array.codecs): assert isinstance(entity, MetadataEntity) - _assert_conforms(entity) + _round_trips(type(entity), entity.to_json()) def test_every_registered_entity_is_checked_here() -> None: @@ -757,6 +747,11 @@ class AcmeShardCache(StorageTransformerEntity): def simplified(self) -> Self: return dataclasses.replace(self, verbose=UNSET) + def to_json(self) -> ZarrV3MetadataFieldJSON: + if self.verbose is UNSET: + return "acme.shard_cache" + return {"name": "acme.shard_cache", "configuration": {"verbose": self.verbose}} + def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> None: # `to_json` is faithful, entities included; `canonical` walks every diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 6549cb21bd..9257c279b7 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -19,7 +19,7 @@ validate_array_metadata_v3, ) from zarr_metadata.v3.codec.blosc import BloscCodec -from zarr_metadata.v3.codec.gzip import GzipCodec, GzipCodecObject +from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.entity import ( CORE_AND_EXTENSIONS, FROM_NAME, @@ -34,6 +34,7 @@ Context, DataTypeEntity, IntegerDataType, + JSONValue, Loc, MetadataEntity, Opaque, @@ -41,6 +42,7 @@ ValidationProblem, ZarrV3MetadataFieldJSON, problem, + written, ) ACME_MAX_ACCELERATION = 65537 @@ -65,6 +67,11 @@ def __post_init__(self) -> None: ) ) + def to_json(self) -> ZarrV3MetadataFieldJSON: + if self.acceleration is UNSET: + return "acme.lz4" + return {"name": "acme.lz4", "configuration": {"acceleration": self.acceleration}} + @dataclass(frozen=True) class AcmeFloat8DataType(DataTypeEntity): @@ -76,6 +83,9 @@ class AcmeFloat8DataType(DataTypeEntity): def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: return () + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.float8" + def _scope() -> Context: return CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec, AcmeFloat8DataType) @@ -258,7 +268,7 @@ def test_error_a_field_may_not_shadow_a_class_variable() -> None: @dataclass(frozen=True) class Negotiable(BytesBytesCodec): - must_understand: bool = True # pyright: ignore[reportIncompatibleVariableOverride] + kind: str = "bytes_bytes" # pyright: ignore[reportIncompatibleVariableOverride] identifier: ClassVar[str] = "acme.negotiable" @@ -273,6 +283,9 @@ def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: class Int24DataType(IntegerDataType): identifier: ClassVar[str] = "acme.int24" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.int24" + # A third-party *family*: one class covering a parameterized set of names, # the way `r` covers every raw-byte width. @@ -340,6 +353,9 @@ class AcmeWrapperCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.wrapper" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return {"name": "acme.wrapper", "configuration": {"inner": written(self.inner)}} + def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> None: # `inner: CodecEntity | Opaque` is the whole declaration. Reading it @@ -427,6 +443,12 @@ class AcmeFramedCodec(BytesBytesCodec): def simplified(self) -> Self: return self if self.frame != 0 else replace(self, frame=UNSET) + def to_json(self) -> ZarrV3MetadataFieldJSON: + configuration: dict[str, JSONValue] = {"inner": written(self.inner)} + if self.frame is not UNSET: + configuration["frame"] = self.frame + return {"name": "acme.framed", "configuration": configuration} + blosc = BloscCodec(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) framed = AcmeFramedCodec(inner=blosc, frame=0) assert framed.canonical() == AcmeFramedCodec(inner=replace(blosc, typesize=UNSET)) @@ -482,6 +504,9 @@ def __post_init__(self) -> None: problem(("block",), f"expected a power of two, got {self.block}", "invalid_value") ) + def to_json(self) -> ZarrV3MetadataFieldJSON: + return {"name": "acme.block", "configuration": {"block": self.block}} + def test_a_rule_about_a_member_is_post_init() -> None: # The rule runs on the typed members and reports relative to the @@ -516,6 +541,9 @@ class AcmeSlotted(BytesBytesCodec): identifier: ClassVar[str] = "acme.slotted" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return {"name": "acme.slotted", "configuration": {"level": self.level}} + assert AcmeSlotted(level=1).to_json() == { "name": "acme.slotted", "configuration": {"level": 1}, @@ -528,6 +556,9 @@ class AcmeNoted(BytesBytesCodec): identifier: ClassVar[str] = "acme.noted" note: ClassVar = "not a member" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.noted" + assert AcmeNoted().to_json() == "acme.noted" @@ -544,10 +575,13 @@ class AcmeScaled(ArrayArrayCodec): def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming + def to_json(self) -> ZarrV3MetadataFieldJSON: + return {"name": "acme.scaled", "configuration": {"scale": self.scale}} + scope = CORE_AND_EXTENSIONS.extended_with(AcmeScaled) - for written in (2, 2.5): + for spelled in (2, 2.5): codec, problems = scope.coerce( - CodecEntity, {"name": "acme.scaled", "configuration": {"scale": written}} + CodecEntity, {"name": "acme.scaled", "configuration": {"scale": spelled}} ) assert problems == () assert isinstance(codec, AcmeScaled) @@ -568,6 +602,9 @@ class Undecorated(BytesBytesCodec): identifier: ClassVar[str] = "acme.undecorated" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return {"name": "acme.undecorated", "configuration": {"level": self.level}} + with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): CORE_AND_EXTENSIONS.extended_with(Undecorated) @@ -590,6 +627,9 @@ def test_error_an_array_array_codec_defines_transition() -> None: class Silent(ArrayArrayCodec): identifier: ClassVar[str] = "acme.silent" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.silent" + with pytest.raises( TypeError, match="does not define transition, which its base leaves abstract" ): @@ -614,6 +654,9 @@ class Lax(DataTypeEntity): identifier: ClassVar[str] = "acme.lax" scalar_storage: ClassVar[StorageClass] = "single_byte" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.lax" + with pytest.raises(TypeError, match="does not define fill_value_problems"): CORE_AND_EXTENSIONS.extended_with(Lax) @@ -636,57 +679,3 @@ def test_error_a_list_of_problem_tuples_is_refused() -> None: # the constructor and fail inside `coerce`, far from the mistake. with pytest.raises(TypeError, match="collect with `extend`, not `append`"): MetadataValidationError([problem(("a",), "bad a")]) # pyright: ignore[reportArgumentType] - - -def test_error_the_named_json_type_must_match_what_the_entity_writes() -> None: - # A required member means the entity is always written as an object, - # so naming a bare-name type for it is a promise `to_json` would break. - with pytest.raises( - TypeError, match="admits a bare name, which the entity never writes; lacks the object" - ): - - @dataclass(frozen=True) - class Misnamed(BytesBytesCodec[Literal["acme.misnamed"]]): - level: int - - identifier: ClassVar[str] = "acme.misnamed" - - -def test_error_the_named_json_type_must_name_what_the_entity_accepts() -> None: - # `GzipCodecObject` spells `name: Literal["gzip"]`; an entity that - # accepts only its own name cannot write that. - with pytest.raises(TypeError, match="names 'gzip', which the entity does not accept"): - - @dataclass(frozen=True) - class Impostor(BytesBytesCodec[GzipCodecObject]): - level: int - - identifier: ClassVar[str] = "acme.impostor" - - -def test_error_the_named_json_type_must_have_the_members_as_keys() -> None: - with pytest.raises( - TypeError, match=r"configuration keys \['lvl'\] where the members are \['level'\]" - ): - - @dataclass(frozen=True) - class Mismatched(BytesBytesCodec[AcmeLvlObject]): - level: int - - identifier: ClassVar[str] = "acme.lvl" - - -def test_a_third_party_entity_may_name_its_json_type_or_not() -> None: - # Left defaulted, `to_json` is typed as any metadata field; named, as - # the entity's own type, held to the members at class creation -- and - # either way the same dict comes back. - @dataclass(frozen=True) - class AcmeTypedBlockCodec(BytesBytesCodec[AcmeBlockObject]): - block: int - - identifier: ClassVar[str] = "acme.block" - - assert AcmeTypedBlockCodec(block=8).to_json() == { - "name": "acme.block", - "configuration": {"block": 8}, - } From f31f9daa41f04c0633cefa525b64e56d93c90415 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 16:08:48 +0200 Subject: [PATCH 083/107] refactor(zarr-metadata): the entity base is not generic An entity names its JSON type once, as the return type of its own `to_json`, and pyright holds the literal to it. Naming it a second time as a type argument of the base bought nothing: the base's `to_json` is abstract, so no method on it ever needed the parameter, and every bare use of a kind (`CodecEntity | Opaque`, a scope's table) wanted the defaulted form anyway. `Generic[JSONT_co]`, the two `TypeIs` helpers the generic base needed to narrow an `object`, and `unsubscripted` go with it; the 38 subscripted bases are plain again. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 3 +- packages/zarr-metadata/changes/4379.misc.2.md | 14 +-- .../src/zarr_metadata/v3/_entity.py | 88 +++++-------------- .../v3/chunk_grid/rectilinear.py | 2 +- .../zarr_metadata/v3/chunk_grid/regular.py | 2 +- .../v3/chunk_key_encoding/default.py | 2 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 2 +- .../src/zarr_metadata/v3/codec/blosc.py | 2 +- .../src/zarr_metadata/v3/codec/bytes.py | 2 +- .../src/zarr_metadata/v3/codec/cast_value.py | 2 +- .../src/zarr_metadata/v3/codec/crc32c.py | 2 +- .../src/zarr_metadata/v3/codec/gzip.py | 2 +- .../zarr_metadata/v3/codec/scale_offset.py | 2 +- .../v3/codec/sharding_indexed.py | 2 +- .../src/zarr_metadata/v3/codec/transpose.py | 2 +- .../src/zarr_metadata/v3/codec/zstd.py | 2 +- .../zarr_metadata/v3/data_type/_families.py | 9 +- .../src/zarr_metadata/v3/data_type/bool.py | 2 +- .../src/zarr_metadata/v3/data_type/bytes.py | 2 +- .../zarr_metadata/v3/data_type/complex128.py | 2 +- .../zarr_metadata/v3/data_type/complex64.py | 2 +- .../src/zarr_metadata/v3/data_type/float16.py | 2 +- .../src/zarr_metadata/v3/data_type/float32.py | 2 +- .../src/zarr_metadata/v3/data_type/float64.py | 2 +- .../src/zarr_metadata/v3/data_type/int16.py | 2 +- .../src/zarr_metadata/v3/data_type/int32.py | 2 +- .../src/zarr_metadata/v3/data_type/int64.py | 2 +- .../src/zarr_metadata/v3/data_type/int8.py | 2 +- .../v3/data_type/numpy_datetime64.py | 2 +- .../v3/data_type/numpy_timedelta64.py | 2 +- .../src/zarr_metadata/v3/data_type/raw.py | 2 +- .../src/zarr_metadata/v3/data_type/string.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 2 +- .../src/zarr_metadata/v3/data_type/uint16.py | 2 +- .../src/zarr_metadata/v3/data_type/uint32.py | 2 +- .../src/zarr_metadata/v3/data_type/uint64.py | 2 +- .../src/zarr_metadata/v3/data_type/uint8.py | 2 +- .../src/zarr_metadata/v3/entity.py | 13 +-- .../tests/v3/test_acme_affine.py | 2 +- .../tests/v3/test_acme_decimal.py | 2 +- 40 files changed, 78 insertions(+), 119 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 403a7ffccc..c1babc6f6e 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -40,8 +40,7 @@ at class creation, and `Annotated[str, FROM_NAME]` marks the one field carried by the envelope's name rather than a configuration key (`r`), which `coerce` fills from the envelope. Whether a configuration is required follows too, since the spec ties the bare-name spelling to whether any member is -required. The public JSON TypedDict an entity names as its base's -argument is held to the fields at class creation, key by key. +required. Nothing a class may restate: every table the layer reads is derived from the fields, and declaring one is refused. Field annotations diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 9cfb9d2bce..f76f0b557b 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -84,12 +84,14 @@ holds which kind is the document's own knowledge, in one place; the entity layer no longer names a document field. `to_json` is abstract: each entity writes its own, as a literal of its -JSON type, and the type checker holds the literal to the TypedDict -- -which is what a base method building a `dict[str, object]` and casting -it to the named type could only assert. The class-creation check that -compared the named type with the fields, the generic `configuration()` -and the rendering walk it needed are gone with the cast; `written` -renders a contained field. +JSON type, which it names as the method's return type -- narrower than +the base's `ZarrV3MetadataFieldJSON` -- and the type checker holds the +literal to that TypedDict, which is what a base method building a +`dict[str, object]` and casting it to the named type could only assert. +The entity base is not generic: nothing is gained by naming the JSON +type twice. The class-creation check that compared the named type with +the fields, the generic `configuration()` and the rendering walk it +needed are gone with the cast; `written` renders a contained field. One thing this does not change, under mypy. An entity's JSON type is a TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index be116184ae..317b963055 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -39,7 +39,6 @@ TYPE_CHECKING, ClassVar, Final, - Generic, Literal, TypeAlias, cast, @@ -49,7 +48,7 @@ get_type_hints, ) -from typing_extensions import TypeIs, TypeVar, is_typeddict +from typing_extensions import TypeVar, is_typeddict from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ( @@ -60,6 +59,7 @@ if TYPE_CHECKING: from typing import Self + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._parts import ArrayParts, ChunkGrid from zarr_metadata.v3._registry import Context @@ -79,7 +79,6 @@ sequence_of, within, ) -from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._compile import ( FROM_NAME, MetadataFieldValue, @@ -98,27 +97,10 @@ shape_of, strip_annotation, type_check, - unsubscripted, ) EntityT = TypeVar("EntityT", bound="MetadataEntity") -JSONT_co = TypeVar( - "JSONT_co", bound=ZarrV3MetadataFieldJSON, default=ZarrV3MetadataFieldJSON, covariant=True -) -"""What an entity's `to_json` returns: its own JSON type, named as the base's argument. - - class GzipCodec(CodecEntity[GzipCodecMetadata]): ... - -Covariant, because it appears only in a return; defaulted, so a bare -`CodecEntity` -- in a field annotation, a table of entities, a scope -- -means `CodecEntity[ZarrV3MetadataFieldJSON]` and admits every codec, each -of whose JSON types is assignable to that one (`ZarrV3NamedConfigJSON` is -`ReadOnly` and closed for exactly this). An entity that leaves it -defaulted is not wrong, only less informative. -""" - - # A real alias, not a string one: entity modules subscript it as # `Coerced[Self]` in a return annotation, and not all of them defer # annotation evaluation. @@ -137,7 +119,6 @@ class GzipCodec(CodecEntity[GzipCodecMetadata]): ... """ -# Left to infer their `Literal` types rather than widened to StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How one scalar of a data type occupies bytes. @@ -155,22 +136,6 @@ class GzipCodec(CodecEntity[GzipCodecMetadata]): ... """ -def is_entity(value: object) -> TypeIs[MetadataEntity]: - """`value` is an entity, of whatever JSON type. - - An `isinstance` against the generic base narrows an `object` to - `MetadataEntity[Unknown]`; this narrows it to the defaulted - `MetadataEntity`, whose `to_json` is any metadata field -- which is - all that can be said of an entity met as an `object`. - """ - return isinstance(value, MetadataEntity) - - -def _is_entity_kind(candidate: object) -> TypeIs[type[MetadataEntity]]: - """`candidate` is an entity class; narrowed as `is_entity` narrows.""" - return isinstance(candidate, type) and issubclass(candidate, MetadataEntity) - - def contains_entity(annotation: object) -> bool: """Whether a value of this type holds a nested metadata field anywhere in it.""" inner, _ = strip_annotation(annotation) @@ -190,23 +155,17 @@ def contains_entity(annotation: object) -> bool: return False -def _as_entity_kind(candidate: object) -> type[MetadataEntity] | None: - """`candidate` as an entity type, or None if it is not one. - - In a function of its own so that the `isinstance`/`issubclass` pair - narrows this parameter and not the caller's variable, which the - caller goes on to read as the annotation it is. - """ - candidate = unsubscripted(candidate) - if _is_entity_kind(candidate): - return candidate +def _entity_class(annotation: object) -> type[MetadataEntity] | None: + """`annotation` when it is an entity class, else None.""" + if isinstance(annotation, type) and issubclass(annotation, MetadataEntity): + return annotation return None def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: """Every entity type an annotation names, at any depth.""" inner, _ = strip_annotation(annotation) - kind = _as_entity_kind(inner) + kind = _entity_class(inner) if kind is not None: return [kind] origin = get_origin(inner) @@ -286,7 +245,7 @@ def written(value: MetadataEntity | Opaque) -> ZarrV3MetadataFieldJSON: def canonicalize_nested(annotation: object, value: object) -> object: """`value` with every nested entity in its own canonical form.""" - if is_entity(value): + if isinstance(value, MetadataEntity): return value.canonical() if isinstance(value, Opaque): return value @@ -598,7 +557,7 @@ def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: @dataclass(frozen=True) -class MetadataEntity(MetadataFieldValue, ABC, Generic[JSONT_co]): +class MetadataEntity(MetadataFieldValue, ABC): """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what @@ -779,7 +738,7 @@ def canonical(self) -> Self: return walked.simplified() @abstractmethod - def to_json(self) -> JSONT_co: + def to_json(self) -> ZarrV3MetadataFieldJSON: """This entity as a document would write it: a literal of its own JSON type. Faithful to every member it holds: read a document, write it @@ -790,10 +749,11 @@ def to_json(self) -> JSONT_co: "configuration": {}}` all read to the same entity, and the entity writes the bare name when every member it holds is absent. - Written per entity, as a literal of the TypedDict named as the - base's argument -- `CodecEntity[GzipCodecObject]` -- which is - what holds it to that type: pyright checks the literal's keys and - values against the TypedDict. A contained entity is written with + Written per entity, as a literal of its own TypedDict and with + that TypedDict as the declared return type -- narrower than the + base's, which is what tells a consumer holding a `GzipCodec` that + it gets a `GzipCodecObject` -- so pyright checks the literal's keys + and values against it. A contained entity is written with `written`. """ @@ -812,7 +772,7 @@ def simplified(self) -> Self: @dataclass(frozen=True) -class CodecEntity(MetadataEntity[JSONT_co], base=True): +class CodecEntity(MetadataEntity, base=True): """An entity that occupies a position in the codec pipeline. Of one of three kinds, each a base class: `ArrayArrayCodec`, @@ -842,7 +802,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb @dataclass(frozen=True) -class ArrayArrayCodec(CodecEntity[JSONT_co], base=True): +class ArrayArrayCodec(CodecEntity, base=True): """A codec that transforms the array: what reaches the next codec is its to say.""" kind: ClassVar[CodecKind] = "array_array" @@ -859,21 +819,21 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: @dataclass(frozen=True) -class ArrayBytesCodec(CodecEntity[JSONT_co], base=True): +class ArrayBytesCodec(CodecEntity, base=True): """The one codec in a pipeline that turns the array into bytes.""" kind: ClassVar[CodecKind] = "array_bytes" @dataclass(frozen=True) -class BytesBytesCodec(CodecEntity[JSONT_co], base=True): +class BytesBytesCodec(CodecEntity, base=True): """A codec that transforms bytes, after the array is gone.""" kind: ClassVar[CodecKind] = "bytes_bytes" @dataclass(frozen=True) -class ChunkGridEntity(MetadataEntity[JSONT_co], base=True): +class ChunkGridEntity(MetadataEntity, base=True): """An entity that divides an array into the parts a pipeline encodes.""" def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: @@ -895,7 +855,7 @@ def grid(self, array_shape: object) -> ChunkGrid: @dataclass(frozen=True) -class DataTypeEntity(MetadataEntity[JSONT_co], base=True): +class DataTypeEntity(MetadataEntity, base=True): """An entity that says how the array's scalars are stored. Only data types answer that, and every rule that turns on it -- a @@ -924,12 +884,12 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class ChunkKeyEncodingEntity(MetadataEntity[JSONT_co], base=True): +class ChunkKeyEncodingEntity(MetadataEntity, base=True): """An entity that says how a chunk's coordinates become a store key.""" @dataclass(frozen=True) -class StorageTransformerEntity(MetadataEntity[JSONT_co], base=True): +class StorageTransformerEntity(MetadataEntity, base=True): """An entity that stands between the codec pipeline and the store.""" @@ -960,7 +920,6 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "CodecKind", "Coerced", "DataTypeEntity", - "JSONT_co", "Loc", "MetadataEntity", "Opaque", @@ -968,7 +927,6 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "StorageTransformerEntity", "TypeCheck", "is_bool", - "is_entity", "is_int", "is_integer", "is_json_value", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index f24053e6db..261f934b8d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -167,7 +167,7 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: @dataclass(frozen=True) -class RectilinearChunkGrid(ChunkGridEntity[RectilinearChunkGridMetadata]): +class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" kind: Literal["inline"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 951131ffb5..fc2e608f64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -61,7 +61,7 @@ class RegularChunkGridObject(TypedDict, closed=True): @dataclass(frozen=True) -class RegularChunkGrid(ChunkGridEntity[RegularChunkGridMetadata]): +class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" chunk_shape: tuple[int, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 6d89dc9dae..9eeabbf75f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -71,7 +71,7 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity[DefaultChunkKeyEncodingMetadata]): +class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity): """The `default` chunk key encoding, coerced from its metadata.""" separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index dd8fcbd157..d0182ffb87 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -77,7 +77,7 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class V2ChunkKeyEncoding(ChunkKeyEncodingEntity[V2ChunkKeyEncodingMetadata]): +class V2ChunkKeyEncoding(ChunkKeyEncodingEntity): """The `v2` chunk key encoding, coerced from its metadata.""" separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index ef6173d5e1..b03f9e8994 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -88,7 +88,7 @@ class BloscCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BloscCodec(BytesBytesCodec[BloscCodecMetadata]): +class BloscCodec(BytesBytesCodec): """The `blosc` codec, coerced from its metadata. Everything blosc knows about itself: the shape its metadata takes, the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 01f8eabec8..6a1fca9749 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -79,7 +79,7 @@ class BytesCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BytesCodec(ArrayBytesCodec[BytesCodecMetadata]): +class BytesCodec(ArrayBytesCodec): """The `bytes` codec, coerced from its metadata. `endian` is optional and absent means something: a one-byte data type diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 76af074139..efbb3d9c6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -127,7 +127,7 @@ class CastValueCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class CastValueCodec(ArrayArrayCodec[CastValueCodecMetadata]): +class CastValueCodec(ArrayArrayCodec): """The `cast_value` codec, coerced from its metadata. Holds the data type it casts to, so like `sharding_indexed` it is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index d5f8635464..8a57edb6e4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -60,7 +60,7 @@ class Crc32cCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class Crc32cCodec(BytesBytesCodec[Crc32cCodecName]): +class Crc32cCodec(BytesBytesCodec): """The `crc32c` codec, coerced from its metadata. The name says everything: a checksum has nothing to configure. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index cdeff9da92..6ff873e5ff 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -66,7 +66,7 @@ class GzipCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class GzipCodec(BytesBytesCodec[GzipCodecMetadata]): +class GzipCodec(BytesBytesCodec): """The `gzip` codec, coerced from its metadata.""" level: int diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index f2dd6967e9..bfe4ce8668 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -74,7 +74,7 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ScaleOffsetCodec(ArrayArrayCodec[ScaleOffsetCodecMetadata]): +class ScaleOffsetCodec(ArrayArrayCodec): """The `scale_offset` codec, coerced from its metadata. Both members are optional and any JSON scalar is well-typed here; what diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index ea2c5295e8..68b279a2b7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -96,7 +96,7 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ShardingIndexedCodec(ArrayBytesCodec[ShardingIndexedCodecMetadata]): +class ShardingIndexedCodec(ArrayBytesCodec): """The `sharding_indexed` codec, coerced from its metadata. Holds two codec pipelines, so it is one of the few entities that diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 9a0b095854..5a379fdf5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -62,7 +62,7 @@ class TransposeCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class TransposeCodec(ArrayArrayCodec[TransposeCodecMetadata]): +class TransposeCodec(ArrayArrayCodec): """The `transpose` codec, coerced from its metadata.""" order: tuple[int, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index f62de7e36f..50ff6ca2f2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -74,7 +74,7 @@ class ZstdCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ZstdCodec(BytesBytesCodec[ZstdCodecMetadata]): +class ZstdCodec(BytesBytesCodec): """The `zstd` codec, coerced from its metadata.""" level: int diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 144497b23f..34cc94e51a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -20,7 +20,6 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( DataTypeEntity, - JSONT_co, StorageClass, is_integer, problem, @@ -65,7 +64,7 @@ def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[Validati @dataclass(frozen=True) -class IntegerDataType(DataTypeEntity[JSONT_co], base=True): +class IntegerDataType(DataTypeEntity, base=True): """A fixed-width integer. The width is the whole difference.""" bounds: ClassVar[tuple[int, int]] @@ -82,7 +81,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class FloatDataType(DataTypeEntity[JSONT_co], base=True): +class FloatDataType(DataTypeEntity, base=True): """A binary float. A fill value may be a number, a named non-finite, or hex.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -122,7 +121,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class ComplexDataType(DataTypeEntity[JSONT_co], base=True): +class ComplexDataType(DataTypeEntity, base=True): """A complex number: a `[real, imag]` pair of the component float type.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -169,7 +168,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class NumpyTimeDataType(DataTypeEntity[JSONT_co], base=True): +class NumpyTimeDataType(DataTypeEntity, base=True): """A numpy time scalar: a signed 64-bit count of units, or `NaT`. The vocabulary the two time types share -- the unit codes and the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index 4eb4d52f15..4ac5e5cbb2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -34,7 +34,7 @@ @dataclass(frozen=True) -class BoolDataType(DataTypeEntity[BoolDataTypeName]): +class BoolDataType(DataTypeEntity): """The `bool` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 3c759ed93d..3e1107ae0a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -60,7 +60,7 @@ def base64_bytes(value: str) -> Base64Bytes: @dataclass(frozen=True) -class BytesDataType(DataTypeEntity[BytesDataTypeName]): +class BytesDataType(DataTypeEntity): """The `bytes` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "variable_length" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 951a6b7850..420aec8caa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -42,7 +42,7 @@ @dataclass(frozen=True) -class Complex128DataType(ComplexDataType[Complex128DataTypeName]): +class Complex128DataType(ComplexDataType): """The `complex128` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index fcf2995896..1e85f0d244 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -42,7 +42,7 @@ @dataclass(frozen=True) -class Complex64DataType(ComplexDataType[Complex64DataTypeName]): +class Complex64DataType(ComplexDataType): """The `complex64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 68c9e26ad8..d537fe9611 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -81,7 +81,7 @@ def hex_float16(value: str) -> HexFloat16: @dataclass(frozen=True) -class Float16DataType(FloatDataType[Float16DataTypeName]): +class Float16DataType(FloatDataType): """The `float16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index f834025dcf..4386750b21 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -81,7 +81,7 @@ def hex_float32(value: str) -> HexFloat32: @dataclass(frozen=True) -class Float32DataType(FloatDataType[Float32DataTypeName]): +class Float32DataType(FloatDataType): """The `float32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 06ad561df5..aad3a99daf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -82,7 +82,7 @@ def hex_float64(value: str) -> HexFloat64: @dataclass(frozen=True) -class Float64DataType(FloatDataType[Float64DataTypeName]): +class Float64DataType(FloatDataType): """The `float64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index 400104a997..6911ca69cc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int16DataType(IntegerDataType[Int16DataTypeName]): +class Int16DataType(IntegerDataType): """The `int16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index 25d78badb8..d900e1b18e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int32DataType(IntegerDataType[Int32DataTypeName]): +class Int32DataType(IntegerDataType): """The `int32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index 9894461834..f76d5a9de2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int64DataType(IntegerDataType[Int64DataTypeName]): +class Int64DataType(IntegerDataType): """The `int64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index fc9feb0879..56780227e8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Int8DataType(IntegerDataType[Int8DataTypeName]): +class Int8DataType(IntegerDataType): """The `int8` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 63d812c360..3b02366051 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -70,7 +70,7 @@ class NumpyDatetime64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyDatetime64DataType(NumpyTimeDataType[NumpyDatetime64]): +class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" unit: NumpyTimeUnit diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index cc4105ee4c..a41bfc433c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -73,7 +73,7 @@ class NumpyTimedelta64(TypedDict, closed=True): @dataclass(frozen=True) -class NumpyTimedelta64DataType(NumpyTimeDataType[NumpyTimedelta64]): +class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" unit: NumpyTimeUnit diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 6750c912c6..e42d3e9b13 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -96,7 +96,7 @@ def _name_problems(name: str) -> tuple[ValidationProblem, ...]: @dataclass(frozen=True) -class RawBytesDataType(DataTypeEntity[RawBytesDataTypeName]): +class RawBytesDataType(DataTypeEntity): """An `r` raw-bytes data type, coerced from its metadata. One class for the whole family, because `r8` and `r4096` differ only diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 9f9b498141..0ce4e9351c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -34,7 +34,7 @@ @dataclass(frozen=True) -class StringDataType(DataTypeEntity[StringDataTypeName]): +class StringDataType(DataTypeEntity): """The `string` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "variable_length" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 308c9fed88..029d7af4df 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -101,7 +101,7 @@ def _written_field(field: StructFieldComponent) -> StructField: @dataclass(frozen=True) -class StructDataType(DataTypeEntity[Struct]): +class StructDataType(DataTypeEntity): """The `struct` data type, coerced from its metadata. A record of named fields, each with a data type of its own -- so this diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 0bbaa52bc9..0bbcd7c603 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint16DataType(IntegerDataType[Uint16DataTypeName]): +class Uint16DataType(IntegerDataType): """The `uint16` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index fa0189ba0d..bce64cd4ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint32DataType(IntegerDataType[Uint32DataTypeName]): +class Uint32DataType(IntegerDataType): """The `uint32` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 57a6446ff9..5948ee211d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint64DataType(IntegerDataType[Uint64DataTypeName]): +class Uint64DataType(IntegerDataType): """The `uint64` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index 7b41b54144..f5a68d2a80 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -29,7 +29,7 @@ @dataclass(frozen=True) -class Uint8DataType(IntegerDataType[Uint8DataTypeName]): +class Uint8DataType(IntegerDataType): """The `uint8` data type. The name says everything.""" scalar_storage: ClassVar[StorageClass] = "single_byte" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 244c8e68d5..cc00015201 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -65,7 +65,7 @@ class AcmeLz4Object(TypedDict, closed=True): configuration: AcmeLz4Configuration @dataclass(frozen=True) # load-bearing: `coerce` builds the entity with cls(**members) - class AcmeLz4Codec(BytesBytesCodec[AcmeLz4Object | Literal["acme.lz4"]]): + class AcmeLz4Codec(BytesBytesCodec): acceleration: int | UNSET = UNSET # optional: defaults to UNSET, never to a value identifier: ClassVar[str] = "acme.lz4" @@ -151,11 +151,12 @@ class creation or registration with a message that says what to write. key is its `identifier`, so `extended_with` takes the classes and nothing can be misfiled. -**Naming the JSON type.** `BytesBytesCodec[AcmeLz4Object | Literal["acme.lz4"]]` -types `to_json` as your own JSON type rather than as any metadata field, -and pyright checks the literal `to_json` returns against it: a key it -does not declare, a required one left out, a value of the wrong type is -a static error. Left unnamed, `to_json` is typed as any metadata field. +**Naming the JSON type.** The return annotation of `to_json` -- above, +`AcmeLz4Object | Literal["acme.lz4"]` -- is the entity's own JSON type, +narrower than the `ZarrV3MetadataFieldJSON` the base declares, and +pyright checks the literal returned against it: a key it does not +declare, a required one left out, a value of the wrong type is a static +error. Two complete extensions written against this module alone, as tests: `tests/v3/test_acme_affine.py` (an `array_array` codec with a number, an diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index d6991d8ecd..c9c956a0b6 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -49,7 +49,7 @@ class AcmeAffineObject(TypedDict, closed=True): @dataclass(frozen=True) -class AcmeAffineCodec(ArrayArrayCodec[AcmeAffineObject]): +class AcmeAffineCodec(ArrayArrayCodec): """`x * scale + offset`, stored as `dtype` if one is named.""" scale: float diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 6a2c79c5b3..39fb1116b2 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -71,7 +71,7 @@ class AcmeDecimal(TypedDict, closed=True): @dataclass(frozen=True) -class AcmeDecimalDataType(DataTypeEntity[AcmeDecimal]): +class AcmeDecimalDataType(DataTypeEntity): """The `acme.decimal` data type, coerced from its metadata.""" precision: int From b8bd1760ab9602e72c7fcc77effc0769ca6aa6e5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 16:31:17 +0200 Subject: [PATCH 084/107] refactor(zarr-metadata): one parser over the fields; canonical and the nested walks are the entity's own The type-annotation reader is one module, `_typed_json`, that knows nothing of entities: `parser_for` turns a field annotation into a parser of JSON values, over the shapes JSON takes and no others. A caller with a shape of its own passes a `leaf`, asked first at every depth. The entity layer's one shape is a field holding another entity, `Kind | Opaque`; its leaf reads the inner entity through the scope and keeps the inner problems apart from the containing entity's own. So a struct's fields' data types and a shard's pipelines are resolved by the same walk that type-checks them, and the three annotation walkers the entity module carried for resolution and canonicalization, with the union-branch and fixed-tuple machinery they needed, are gone. `canonical` is the entity's own, as `to_json` is: the entity itself by default, overridden where two spellings mean the same and, in an entity that contains entities, to put those in canonical form with `canonicalized`. No base walk, no `simplified` hook, no `@final`. Class creation refuses two things: a field whose annotation is not a shape JSON takes, and a class variable a base annotates and nothing sets. The other nine invariants either duplicated pyright (a field shadowing a class variable, a `Literal` class variable outside its values, an override of a final method), duplicated registration (an entity of no kind; a codec skipping the kind classes, now refused there), or guarded defaults `coerce` no longer relies on: an optional member the document left out is passed as `UNSET` explicitly. A codec's pipeline position is its base class alone; the `kind` string it also carried is gone, and the chain rules ask `isinstance`. One diagnostic is more complete: when one element of a shard's pipeline is not a metadata field at all, the other elements are still read and judged, where before the whole member stood down. Over the 40,000 document corpus that adds 133 problems to 91 documents already invalid and changes no verdict; nothing is lost. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.10.md | 11 +- .../zarr-metadata/changes/4379.feature.7.md | 60 +- .../zarr-metadata/changes/4379.feature.8.md | 5 +- packages/zarr-metadata/changes/4379.misc.1.md | 5 +- packages/zarr-metadata/changes/4379.misc.2.md | 61 +- .../src/zarr_metadata/v3/_chain.py | 18 +- .../src/zarr_metadata/v3/_checks.py | 227 ----- .../src/zarr_metadata/v3/_compile.py | 540 ----------- .../src/zarr_metadata/v3/_document.py | 29 +- .../src/zarr_metadata/v3/_entity.py | 850 ++++++------------ .../src/zarr_metadata/v3/_registry.py | 25 +- .../src/zarr_metadata/v3/_typed_json.py | 649 +++++++++++++ .../v3/chunk_grid/rectilinear.py | 2 +- .../src/zarr_metadata/v3/codec/__init__.py | 7 +- .../src/zarr_metadata/v3/codec/blosc.py | 4 +- .../src/zarr_metadata/v3/codec/cast_value.py | 9 +- .../v3/codec/sharding_indexed.py | 13 +- .../src/zarr_metadata/v3/data_type/struct.py | 14 +- .../src/zarr_metadata/v3/entity.py | 43 +- .../tests/rules/test_chain_properties.py | 11 +- .../zarr-metadata/tests/test_public_api.py | 1 - .../tests/v3/test_acme_affine.py | 11 +- .../zarr-metadata/tests/v3/test_entities.py | 2 +- .../tests/v3/test_extension_api.py | 133 ++- 24 files changed, 1168 insertions(+), 1562 deletions(-) delete mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_checks.py delete mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_compile.py create mode 100644 packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index c88e61a831..bc540048e7 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -36,9 +36,8 @@ valid under both reports identically. Among those invalid under both, 4,397 report fewer problems and 15,259 report more, the latter because a problem that used to stand down the rest of an entity no longer does. -Registering an entity is checked at class creation, because every way of -getting it wrong type-checks cleanly and then fails somewhere that will -not name the class: a class variable the entity owes and did not declare -(read off the annotations, so a family adding one cannot forget to -require it), a field shadowing one, and a member whose default -contradicts whether the spec requires it. +Two ways of writing an entity type-check cleanly and then fail somewhere +that will not name the class, so class creation refuses them: a field +whose annotation is not a shape JSON takes, and a class variable the +entity owes and did not declare (read off the annotations, so a family +adding one cannot forget to require it). diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index c1babc6f6e..6a165ab0a1 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -30,11 +30,12 @@ because there is no honest reading of a `blosc` whose level is a string. The type judgment is read off the entity's own fields rather than written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field -annotations: `int`, `bool`, `str`, a `Literal` of names, the JSON-value -alias, an array homogeneous or fixed, a union of those, a nested object -described by a TypedDict or a record dataclass, an object of undeclared -keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested -metadata field -- an entity type, with or without `Opaque`. That covers +annotations: `int`, `float` for any number, `bool`, `str`, a `Literal` +of names, the JSON-value alias, an array homogeneous or fixed, a union +of those, a nested object described by a TypedDict or a record +dataclass, an object of undeclared keys as `Mapping[str, V]`, a +`NewType` as the type it names, and a nested metadata field, written as +its kind with `Opaque`. That covers every member this package models; an annotation outside them is refused at class creation, and `Annotated[str, FROM_NAME]` marks the one field carried by the envelope's name rather than a configuration key (`r`), @@ -42,35 +43,30 @@ which `coerce` fills from the envelope. Whether a configuration is required follows too, since the spec ties the bare-name spelling to whether any member is required. -Nothing a class may restate: every table the layer reads is derived -from the fields, and declaring one is refused. Field annotations -are resolved per class, skipping class variables by text, so a -`ClassVar` naming something imported only for the type checker cannot -fail class creation. +Nothing is derived from the fields ahead of time: `coerce` parses a +configuration against them as it reads it. Field annotations are +resolved per class, skipping class variables by text, so a `ClassVar` +naming something imported only for the type checker cannot fail class +creation. One diagnostic became more precise: a malformed `[value, count]` pair in a rectilinear grid is now reported at the offending element inside the pair rather than at the pair. No verdict changes. -An entity that contains other entities writes nothing for it. A field -annotated with an entity type -- `data_type: DataTypeEntity | Opaque`, -`codecs: tuple[CodecEntity | Opaque, ...]`, a record holding one -- is -resolved through the scope at the point that kind is registered at, -written back as each contained entity's own JSON, and put in canonical -form by recursing into it, all read off the annotation. The `prepare` -hook is gone, and so are the three `prepare`, three `configuration` and -three `canonical` overrides that did that walk by hand for `cast_value`, -`sharding_indexed` and `struct`. Each entity kind names its -`extension_point`, which is what makes a nested field resolvable; a -field typed as bare `MetadataEntity` is refused at class creation, since -no scope could place it. - -That walk is `canonical`'s alone. An entity's own rewrite -- two -spellings of its own members that mean the same, a rectilinear -dimension's run-length encoding, a `typesize` that `noshuffle` ignores --- goes in `simplified`, a hook `canonical` calls after the walk; an -override of `canonical` itself is refused at class creation, so the -walk cannot be lost and there is no `super()` to remember. +An entity that contains other entities declares the field -- +`data_type: DataTypeEntity | Opaque`, `codecs: tuple[CodecEntity | +Opaque, ...]`, a record holding one -- and the contained entity is read +through the scope the containing one is read in, at whatever depth the +annotation puts it. Writing it back and putting it in canonical form +are the containing entity's own two lines, `written(self.inner)` in +`to_json` and `replace(self, inner=canonicalized(self.inner))` in +`canonical`, the same way it writes and simplifies its own members. +`canonical` is the entity itself by default, overridden where two +spellings of its members mean the same -- a rectilinear dimension's +run-length encoding, a `typesize` that `noshuffle` ignores. A field +typed as bare `MetadataEntity`, or as an entity without `Opaque`, is +refused at class creation: no scope could place the one, and the other +lies about what the field holds when the inner name is out of scope. Everything finer than a type -- a bound, a rule about one member, a rule that reads two members together -- is the entity's own @@ -81,7 +77,9 @@ error and reports the problems in the document instead. The space of refinements is too wide to capture statically, and reading them in one place per entity is what will show where a shared form is worth having. -The set of annotation shapes the compiler reads is closed: the shapes +The set of annotation shapes the parser reads is closed: the shapes JSON takes, and no others. A field annotation outside them is refused at class creation, and the field is written as one of them instead, with -any finer rule in `__post_init__`. +any finer rule in `__post_init__`. The parser is one module that knows +nothing of entities; the entity layer hands it the one shape of its own, +a field holding an entity, as a leaf it reads at any depth. diff --git a/packages/zarr-metadata/changes/4379.feature.8.md b/packages/zarr-metadata/changes/4379.feature.8.md index 0df75e570e..a7bd2d6d74 100644 --- a/packages/zarr-metadata/changes/4379.feature.8.md +++ b/packages/zarr-metadata/changes/4379.feature.8.md @@ -31,9 +31,8 @@ Two properties are asserted over generated documents: canonicalizing twice changes nothing further, and canonicalizing never changes a verdict. The document owns its canonical form. `ArrayDocumentV3.canonical()` -walks the fields that hold an entity the way every entity containing -entities does -- `storage_transformers` included, which the hand-written -walk it replaces never reached -- and applies the one rule that is the +puts each of its five extension points in canonical form -- +`storage_transformers` included -- and applies the one rule that is the document's own. `canonicalize_array_metadata_v3` is the door onto it, and reads the document once: the entities that judge it are the entities that are rewritten. diff --git a/packages/zarr-metadata/changes/4379.misc.1.md b/packages/zarr-metadata/changes/4379.misc.1.md index 0425c9471b..2887f26117 100644 --- a/packages/zarr-metadata/changes/4379.misc.1.md +++ b/packages/zarr-metadata/changes/4379.misc.1.md @@ -18,8 +18,9 @@ allowed follows from the TypedDict's required keys. One more table went the same way: `zarr_metadata.v3.codec.kind`, which sorted codec *names* into the spec's three pipeline kinds. A name does -not have a pipeline position, a codec does, and each one now declares -its own `kind`. Removed with it: `codec_kind_of_name`, +not have a pipeline position, a codec does, and each one now says so by +its base class: `ArrayArrayCodec`, `ArrayBytesCodec` or +`BytesBytesCodec`. Removed with it: `codec_kind_of_name`, `ARRAY_ARRAY_CODEC_NAMES`, `ARRAY_BYTES_CODEC_NAMES`, `BYTES_BYTES_CODEC_NAMES`, and `codec.blosc.canonical_configuration`, whose one job is `BloscCodec.canonical()`. diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index f76f0b557b..bebc637976 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -21,27 +21,24 @@ against the rule registry this layer replaced, over one shared corpus of 40,000 documents, no verdict differs in the laxer direction and the valid documents report identically. -An entity names its own JSON type as the base's argument -- -`class GzipCodec(CodecEntity[GzipCodecMetadata])` -- and `to_json` returns -it. The parameter has a default, so a bare `CodecEntity` -- in a field, a -table, a scope -- admits every codec, and a third party may leave it -unnamed. -`crc32c` names `Crc32cCodecName` alone: it has no members, so it only -ever writes the bare name, and the union is what a document may spell, -not what the entity writes. For the conformance test to use the -package's own compiler as its oracle, `check_for` now reads two shapes -it did not: `Mapping[str, V]`, an object of undeclared keys, and a -`NewType`, as the type it names -- which also closes a gap for a -third-party field typed with either. +An entity's `to_json` names its own JSON type as its return type -- +`GzipCodecObject` for `GzipCodec`; `Crc32cCodecName` alone for `crc32c`, +which has no members and only ever writes the bare name -- and the +entity base is not generic. For the conformance test to use the +package's own parser as its oracle, it reads two shapes it did not: +`Mapping[str, V]`, an object of undeclared keys, and a `NewType`, as the +type it names -- which also closes a gap for a third-party field typed +with either. -Class creation is two steps with one owner each: `_compile_entity` -derives the tables the layer reads from the fields, and `_INVARIANTS`, -a declared tuple of named functions, asks each invariant of the -compiled class in order. `canonical` is `@final`, so an override is -refused by pyright in the author's editor as well as at class creation; the guards that refused `problems` and `prepare` -- -names from earlier drafts of this branch, never released -- are gone, -and nothing is kept on the class: what the layer needs of an entity's -fields, it reads off them when it reads a document. +Class creation refuses two things and no more: a field whose annotation +is not a shape JSON takes, and a class variable a base annotates and +nothing sets. Everything else an author could get wrong, pyright says +in the editor -- a field shadowing a class variable, a `Literal` class +variable outside its values -- or registration says, with what to +write: no `@dataclass`, a kind's abstract method left undefined, a +codec subclassing `CodecEntity` instead of a kind. Nothing is kept on +the class: what the layer needs of an entity's fields, it reads off +them when it reads a document. Four adversarial reviews -- two extension authors writing a codec and a data type against the door alone, a design review, an onboarding review @@ -52,8 +49,8 @@ problems come out in the same pass; `FROM_NAME` fields are filled by the base; `to_json` on a document writes only the fields it had; `float` is a shape. Class creation and registration refuse the mistakes both authors made or nearly made -- no `@dataclass`, a nested field without -`Opaque`, an `array_array` codec without `transition`, a `Literal`-typed -class variable outside its values, a list of `problem()` tuples -- with +`Opaque`, an `array_array` codec without `transition`, a list of +`problem()` tuples -- with messages that say what to write. The door's docstring is now the guide its example claimed to be, runnable as written, with the composition contract per kind and what comes back; it exports what its example @@ -70,10 +67,22 @@ must answer is abstract on it -- `transition` on `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`) rather than a string it sets, and an entity that leaves a hook undefined is refused at registration rather than accepted with a silent default. Nothing is derived from an -entity's fields ahead of time: `coerce` type-checks a configuration -against them as it reads it. The compiler -recognises a nested metadata field by a marker base, `MetadataFieldValue`, -rather than by a module attribute set from outside. +entity's fields ahead of time: `coerce` parses a configuration against +them as it reads it. The parser is one module that knows nothing of +entities; the entity layer hands it the one shape of its own -- a field +holding an entity, `Kind | Opaque` -- as a leaf it reads at any depth, +so a struct's fields' data types and a shard's pipelines are read by +the same walk that type-checks them, and no second walk over the +annotations exists. + +`canonical` is the entity's own, as `to_json` is: the entity itself by +default, overridden where two spellings mean the same and, in an entity +that contains entities, to put those in canonical form with +`canonicalized`. One diagnostic is more complete: when one element of a +shard's pipeline is not a metadata field at all, the other elements are +still read and judged, where before the whole member stood down. Over +the 40,000-document corpus that adds 133 problems to 91 documents +already invalid, and changes no verdict. A scope holds entities by kind -- `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity`, `CodecEntity`, `StorageTransformerEntity` -- diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py index 0b4bd39f03..aa926e68a1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.v3._entity import ArrayArrayCodec, CodecEntity, within +from zarr_metadata.v3._entity import ArrayArrayCodec, ArrayBytesCodec, CodecEntity, within if TYPE_CHECKING: from collections.abc import Sequence @@ -31,7 +31,14 @@ from zarr_metadata.v3._entity import Loc, Opaque from zarr_metadata.v3._parts import ArrayParts -_KIND_RANK = {"array_array": 0, "array_bytes": 1, "bytes_bytes": 2} + +def _stage(codec: CodecEntity) -> tuple[int, str]: + """Where in the pipeline a codec stands, as a rank and as the spec names it.""" + if isinstance(codec, ArrayArrayCodec): + return 0, "array->array" + if isinstance(codec, ArrayBytesCodec): + return 1, "array->bytes" + return 2, "bytes->bytes" def _label(codec: CodecEntity | Opaque) -> str: @@ -56,19 +63,18 @@ def order_problems( for index, codec in enumerate(codecs): if not isinstance(codec, CodecEntity): continue - kind = type(codec).kind - rank = _KIND_RANK[kind] + rank, stage = _stage(codec) if rank < latest: problems.append( ValidationProblem( (*loc, index), - f"{kind.replace('_', '->')} codec {_label(codec)} may not " + f"{stage} codec {_label(codec)} may not " "follow a later-stage codec in the pipeline", "invalid_value", ) ) latest = max(latest, rank) - if kind == "array_bytes": + if isinstance(codec, ArrayBytesCodec): array_bytes += 1 if array_bytes > 1: problems.append( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py b/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py deleted file mode 100644 index a9762b624a..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_checks.py +++ /dev/null @@ -1,227 +0,0 @@ -"""The member checks every entity needs. - -One check is a function of a value and its location that returns the -problems it found -- none, for a value of the right type. The scalars, -a closed set of names, a homogeneous sequence, an object, and a nested -metadata field cover what a configuration member can be. `within` and -`named_configuration` are how an entity's problems and envelope are read -from the document that holds it. -""" - -from __future__ import annotations - -# Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below -# (`TypeCheck`) are resolved by `get_type_hints` at class -# creation, and a name that exists only for the type checker is a NameError -# then -- for this package and for any tool introspecting an entity. -from collections.abc import Callable, Mapping, Sequence -from typing import ( - TYPE_CHECKING, - TypeAlias, - cast, -) - -from typing_extensions import TypeIs - -from zarr_metadata.model._validation import ( - ValidationProblem, - is_json, -) - -if TYPE_CHECKING: - from zarr_metadata.model._validation import ProblemKind - - -Loc: TypeAlias = "tuple[str | int, ...]" - - -TypeCheck: TypeAlias = "Callable[[object, Loc], tuple[ValidationProblem, ...]]" -"""Whether one value has the type a member declares, and where if not.""" - - -def problem( - loc: Loc, message: str, kind: ProblemKind = "invalid_type" -) -> tuple[ValidationProblem, ...]: - """One problem, as the one-element tuple every check returns. - - A tuple so that a check can return it directly and a rule can - `found.extend(problem(...))` and raise `MetadataValidationError(found)` - once. The default `kind` names a type mismatch; a value rule passes - `"invalid_value"`. - """ - return (ValidationProblem(loc, message, kind),) - - -def is_integer(value: object) -> TypeIs[int]: - """A JSON integer: an `int`, and not a `bool`. - - `True` is an `int` in Python and `true` is not a number in JSON, so - the two have to be told apart everywhere a number is expected. - """ - return not isinstance(value, bool) and isinstance(value, int) - - -def is_int(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """An integer, and not a bool -- JSON `true` is not the integer 1.""" - if not is_integer(value): - return problem(loc, f"expected an integer, got {value!r}") - return () - - -def is_str(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, str): - return problem(loc, f"expected a string, got {value!r}") - return () - - -def is_bool(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, bool): - return problem(loc, f"expected a boolean, got {value!r}") - return () - - -def is_number(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """A JSON number: an `int` or a `float`, and not a `bool`.""" - if isinstance(value, bool) or not isinstance(value, (int, float)): - return problem(loc, f"expected a number, got {value!r}") - return () - - -def is_json_value(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """Any JSON value at all -- the widest type a member can declare.""" - if not is_json(value): - return problem(loc, f"expected a JSON value, got {value!r}") - return () - - -def one_of(allowed: tuple[str, ...]) -> TypeCheck: - """A member whose type is a closed set of names.""" - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if value not in allowed: - return problem(loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value") - return () - - return check - - -def sequence_of(element: TypeCheck) -> TypeCheck: - """A member whose type is a sequence, checked element by element.""" - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, (list, tuple)): - return problem(loc, f"expected a sequence, got {value!r}") - elements: tuple[object, ...] = tuple(cast("list[object] | tuple[object, ...]", value)) - return tuple( - found for index, entry in enumerate(elements) for found in element(entry, (*loc, index)) - ) - - return check - - -def object_of(value: TypeCheck) -> TypeCheck: - """A member whose type is an object with any keys, checked value by value. - - The open counterpart of `mapping_of`: a `Mapping[str, V]` says nothing - about which keys there are, only what each value must be. - """ - - def check(candidate: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(candidate, Mapping): - return problem(loc, f"expected an object, got {candidate!r}") - entries = cast("Mapping[str, object]", candidate) - return tuple(found for key, entry in entries.items() for found in value(entry, (*loc, key))) - - return check - - -def as_tuples(value: object) -> object: - """Every JSON array in `value`, at any depth, as a tuple. - - The TypedDicts spell a JSON array as a tuple throughout, so a member - taken straight from parsed JSON would otherwise hold a list where its - own type says tuple -- and two documents differing only in that would - compare unequal. - """ - if isinstance(value, (list, tuple)): - entries = cast("list[object] | tuple[object, ...]", value) - return tuple(as_tuples(entry) for entry in entries) - if isinstance(value, Mapping): - entries = cast("Mapping[str, object]", value) - return {key: as_tuples(entry) for key, entry in entries.items()} - return value - - -def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - """A nested metadata field: a bare name or a named-configuration object. - - Only the envelope's shape. Which entity the name denotes, and whether - its configuration is well formed, is settled when the containing - entity reads it in scope. - """ - if not isinstance(value, (str, Mapping)): - return problem(loc, f"expected a metadata field, got {value!r}") - return () - - -def within(prefix: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: - """One entity's problems, located in the document that holds it. - - An entity reports relative to its own `configuration`, so that is what - goes between the field and the member. A problem with an empty - location is about the entity itself -- a malformed `r` name, a - codec that cannot encode what reaches it -- and lands on the field. - """ - return tuple( - ValidationProblem( - (*prefix, *(("configuration", *found.loc) if len(found.loc) != 0 else ())), - found.message, - found.kind, - ) - for found in problems - ) - - -def named_configuration( - value: object, -) -> tuple[str | None, Mapping[str, object] | None, bool]: - """Split metadata into `(name, configuration, must_understand)`. - - The shared shape every entity arrives in: a bare name, or an object - carrying one. A `None` name means the value is not a metadata field at - all; a `None` configuration means the bare spelling was used. - """ - if isinstance(value, str): - return value, None, True - if not isinstance(value, Mapping): - return None, None, True - entry = cast("Mapping[str, object]", value) - name = entry.get("name") - if not isinstance(name, str): - return None, None, True - configuration = entry.get("configuration") - must_understand = entry.get("must_understand", True) - return ( - name, - cast("Mapping[str, object]", configuration) if isinstance(configuration, Mapping) else None, - must_understand if isinstance(must_understand, bool) else True, - ) - - -__all__ = [ - "Loc", - "TypeCheck", - "as_tuples", - "is_bool", - "is_int", - "is_integer", - "is_json_value", - "is_metadata_field", - "is_number", - "is_str", - "named_configuration", - "one_of", - "problem", - "sequence_of", - "within", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py b/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py deleted file mode 100644 index f21528f63e..0000000000 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_compile.py +++ /dev/null @@ -1,540 +0,0 @@ -"""What a field annotation says, read off it: the type. - -An entity's dataclass fields are its schema, and this module is the -compiler over them. `check_for` turns an annotation into its type check --- a scalar, a `Literal`, arrays homogeneous or fixed, unions, a nested -object described by a TypedDict or a record dataclass, an object of -undeclared keys, a `NewType` as the type it names: the shapes JSON takes -and no others, which is what keeps it small. Anything finer than a type --- a bound, a rule about a member, members read together -- is the -entity's own `__post_init__`, in plain code. - -Nothing here knows what an entity is. A nested metadata field is a -field typed as a class deriving from `MetadataFieldValue`, which is the -one thing the compiler is told about them. -""" - -from __future__ import annotations - -# Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below -# (`TypeCheck`) are resolved by `get_type_hints` at class -# creation, and a name that exists only for the type checker is a NameError -# then -- for this package and for any tool introspecting an entity. -import sys -import types -from collections.abc import Mapping, Sequence -from dataclasses import is_dataclass -from typing import ( - TYPE_CHECKING, - Annotated, - ClassVar, - Final, - Literal, - NewType, - NotRequired, - Required, - Union, - cast, - get_args, - get_origin, - get_type_hints, -) - -from typing_extensions import ReadOnly, TypeIs, is_typeddict - -from zarr_metadata._common import JSONValue -from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.v3._checks import ( - is_bool, - is_int, - is_integer, - is_json_value, - is_metadata_field, - is_number, - is_str, - object_of, - one_of, - problem, - sequence_of, -) - -if TYPE_CHECKING: - from zarr_metadata.model._validation import ValidationProblem - from zarr_metadata.v3._checks import Loc, TypeCheck - - -class _FromName: - """The marker behind `FROM_NAME`.""" - - __slots__ = () - - def __repr__(self) -> str: - return "FROM_NAME" - - -FROM_NAME: Final = _FromName() -"""Marks a field carried by the metadata envelope's `name`, not its configuration. - - data_type_name: Annotated[str, FROM_NAME] - -A member all the same -- `value_problems` judges it -- but not a -configuration key, so it is neither read from nor written to a -`configuration` object. The raw-bytes family is the case: `r` keeps its -width in its name and has no configuration at all. -""" - - -def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: - """An annotation's type, and the metadata `Annotated` wrapped it in. - - `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict - key, not part of the value's type; peeled with the `Annotated` layers, - in whatever order they were written. - """ - metadata: list[object] = [] - while True: - origin = get_origin(annotation) - if origin is Annotated: - inner, *extras = get_args(annotation) - metadata.extend(extras) - annotation = inner - elif origin in (NotRequired, Required, ReadOnly): - (annotation,) = get_args(annotation) - else: - return annotation, tuple(metadata) - - -def own_annotations(klass: type) -> dict[str, object]: - """A class's own annotations, unevaluated. - - From 3.14 a class does not carry an `__annotations__` dict until it is - asked for one, and asking evaluates every annotation at once -- so a - `ClassVar` naming something imported only for the type checker would - fail the whole class. `annotationlib` can hand them back as the text - they were written as, which is what the callers here want anyway: - class variables are skipped by text before anything is evaluated. - Earlier versions leave the dict on the class, strings or values as - the module chose. - """ - if sys.version_info >= (3, 14): - import annotationlib - - return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) - return dict(vars(klass).get("__annotations__", {})) - - -def field_hints(cls: type) -> dict[str, object]: - """The dataclass fields of `cls`, resolved, base first. - - Each class's own annotations are resolved in that class's module, - and class variables are skipped *before* resolving, by text -- so a - `ClassVar` whose annotation names something imported only for the - type checker cannot fail class creation. `@dataclass` sees the same - set, in the same order. - """ - hints: dict[str, object] = {} - for ancestor in reversed(cls.__mro__): - raw = { - name: annotation - for name, annotation in own_annotations(ancestor).items() - if not is_class_var(annotation) - } - if len(raw) == 0: - continue - shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) - hints.update(get_type_hints(shell, include_extras=True)) - return hints - - -def is_union(annotation: object) -> bool: - return get_origin(annotation) in (Union, types.UnionType) - - -def is_optional(annotation: object) -> bool: - """Whether a field may be absent: its type admits `UNSET`.""" - inner, _ = strip_annotation(annotation) - return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) - - -class MetadataFieldValue: - """What a metadata field holds once read: an entity, or the JSON it could not read. - - A base with no behaviour. A field annotated with a class deriving - from it -- `CodecEntity | Opaque` -- is a nested metadata field, which - is all the compiler needs to know of entities. - """ - - __slots__ = () - - -def unsubscripted(candidate: object) -> object: - """`CodecEntity[X]` as `CodecEntity`; anything else as it is.""" - origin = get_origin(candidate) - return origin if isinstance(origin, type) else candidate - - -def is_metadata_field_type(candidate: object) -> TypeIs[type[MetadataFieldValue]]: - """Whether `candidate` is a class a metadata field may hold a value of.""" - candidate = unsubscripted(candidate) - return isinstance(candidate, type) and issubclass(candidate, MetadataFieldValue) - - -def is_nested_field(annotation: object) -> bool: - """A metadata-field class, or a union of them (with `UNSET`, if optional).""" - candidates = [ - candidate - for candidate in (get_args(annotation) if is_union(annotation) else (annotation,)) - if candidate is not UNSET - ] - return len(candidates) != 0 and all( - is_metadata_field_type(candidate) for candidate in candidates - ) - - -def describe(annotation: object) -> str: - """The annotation as a message would name it: "an integer", "an object".""" - inner, _ = strip_annotation(annotation) - if is_nested_field(inner): - return "a metadata field" - if inner is int: - return "an integer" - if inner is float: - return "a number" - if inner is bool: - return "a boolean" - if inner is str: - return "a string" - if inner is JSONValue: - return "a JSON value" - origin = get_origin(inner) - if origin is Literal: - return f"one of {tuple(sorted(get_args(inner)))!r}" - if is_union(inner): - branches = [arg for arg in get_args(inner) if arg is not UNSET] - return " or ".join(describe(branch) for branch in branches) - if origin is tuple: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - return f"an array of {describe(arguments[0])} elements" - if len(arguments) == 2: - return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" - return f"an array of {len(arguments)} elements" - if origin in (Mapping, dict): - return "an object" - if isinstance(inner, NewType): - return describe(inner.__supertype__) - if is_typeddict(inner) or is_dataclass(inner): - return "an object" - return "a value" - - -def shape_of(annotation: object) -> str | None: - """The top-level JSON shape an annotation admits, for choosing a union branch. - - None means any shape -- a JSON value, or a union that mixes them. - """ - inner, _ = strip_annotation(annotation) - if is_nested_field(inner): - return "field" - if inner is int: - return "int" - if inner is float: - return "number" - if inner is bool: - return "bool" - if inner is str: - return "str" - origin = get_origin(inner) - if origin is Literal: - values = get_args(inner) - return "int" if all(isinstance(value, int) for value in values) else "str" - if origin is tuple: - return "tuple" - if origin in (Mapping, dict): - return "mapping" - if isinstance(inner, NewType): - return shape_of(inner.__supertype__) - if is_typeddict(inner) or is_dataclass(inner): - return "mapping" - return None - - -def has_shape(shape: str | None, value: object) -> bool: - if shape is None: - return True - if shape == "int": - return is_integer(value) - if shape == "number": - return not isinstance(value, bool) and isinstance(value, (int, float)) - if shape == "bool": - return isinstance(value, bool) - if shape == "str": - return isinstance(value, str) - if shape == "tuple": - return isinstance(value, tuple) - if shape == "mapping": - return isinstance(value, Mapping) - return isinstance(value, (str, Mapping)) # "field" - - -def any_of(branches: Sequence[tuple[object, TypeCheck]], description: str) -> TypeCheck: - """A member whose type is a union of shapes, judged by the branch it fits. - - The branch whose top-level shape the value has is the one that - reports -- so an element inside a malformed array is located inside - the array, rather than the whole array being called wrong. A value - fitting no branch's shape is reported once, by what was expected. - """ - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - fitting = [ - check for annotation, check in branches if has_shape(shape_of(annotation), value) - ] - if len(fitting) == 0: - return problem(loc, f"expected {description}, got {value!r}") - verdicts = [check(value, loc) for check in fitting] - return () if any(len(verdict) == 0 for verdict in verdicts) else verdicts[0] - - return check - - -def fixed_tuple(elements: Sequence[TypeCheck], description: str) -> TypeCheck: - """A member whose type is an array of a fixed length, checked position by position.""" - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, tuple) or len(cast("tuple[object, ...]", value)) != len(elements): - return problem(loc, f"expected {description}, got {value!r}") - entries = cast("tuple[object, ...]", value) - return tuple( - found - for position, (element, entry) in enumerate(zip(elements, entries, strict=True)) - for found in element(entry, (*loc, position)) - ) - - return check - - -def mapping_of(members: Mapping[str, tuple[bool, TypeCheck]]) -> TypeCheck: - """A member that is itself an object with declared keys, checked key by key. - - Closed, like every configuration in this package: a key the type does - not declare is `unknown_key`, a required one missing is `missing_key`, - both located at the object. Each present member is checked at its own - key, so a problem inside is located inside. - """ - - def check(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: - if not isinstance(value, Mapping): - return problem(loc, f"expected an object, got {value!r}") - entries = cast("Mapping[str, object]", value) - found: list[ValidationProblem] = [] - for key in entries: - if key not in members: - found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) - for key, (required, member) in members.items(): - if key not in entries: - if required: - found.extend(problem(loc, f"missing required key {key!r}", "missing_key")) - continue - found.extend(member(entries[key], (*loc, key))) - return tuple(found) - - return check - - -def _members_of(annotations: Mapping[str, object]) -> dict[str, tuple[bool, TypeCheck]] | None: - """A member table for a nested object's keys; None if any key's type has no check.""" - members: dict[str, tuple[bool, TypeCheck]] = {} - for key, annotation in annotations.items(): - check = check_for(annotation) - if check is None: - return None - inner, _ = strip_annotation(annotation) - required = get_origin(annotation) is not NotRequired and not is_optional(inner) - members[key] = (required, check) - return members - - -def _compile_literal(inner: object) -> TypeCheck | None: - # Sorted, because the order `get_args` reports is not the order the - # `Literal` was written in: two `Literal`s over the same values - # compare and hash equal, so the first one built anywhere in the - # process is the one every later one resolves to. The check is a - # membership test either way; this is so the message listing the - # values does not depend on import order. - return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) - - -def _compile_union(inner: object) -> TypeCheck | None: - branches = [arg for arg in get_args(inner) if arg is not UNSET] - if len(branches) == 1: - return check_for(branches[0]) - compiled = [(branch, check_for(branch)) for branch in branches] - if any(check is None for _, check in compiled): - return None - return any_of( - [(branch, cast("TypeCheck", check)) for branch, check in compiled], describe(inner) - ) - - -def _compile_tuple(inner: object) -> TypeCheck | None: - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - element = check_for(arguments[0]) - return None if element is None else sequence_of(element) - elements = [check_for(argument) for argument in arguments] - if any(element is None for element in elements): - return None - return fixed_tuple([cast("TypeCheck", element) for element in elements], describe(inner)) - - -def _compile_typeddict(inner: object) -> TypeCheck | None: - members = _members_of(get_type_hints(inner, include_extras=True)) - return None if members is None else mapping_of(members) - - -def _compile_record(inner: object) -> TypeCheck | None: - if not isinstance(inner, type): # pragma: no cover - the predicate says it is - return None - members = _members_of(field_hints(inner)) - return None if members is None else mapping_of(members) - - -def _compile_mapping(inner: object) -> TypeCheck | None: - # An object of undeclared keys: `Mapping[str, V]`, every value a `V`. - arguments = get_args(inner) - if len(arguments) != 2 or arguments[0] is not str: - return None - value = check_for(arguments[1]) - return None if value is None else object_of(value) - - -def _compile_new_type(inner: object) -> TypeCheck | None: - # A `NewType` is its supertype to a document; the distinction is the - # code's, for a value it has vouched for. - return check_for(cast("NewType", inner).__supertype__) - - -def is_from_name(annotation: object) -> bool: - """Whether `FROM_NAME` marks the field: carried by the envelope's name, not a configuration key.""" - return any(entry is FROM_NAME for entry in strip_annotation(annotation)[1]) - - -def type_check(annotation: object) -> TypeCheck: - """The type check a field annotation implies; `TypeError` if it implies none.""" - check = check_for(annotation) - if check is None: - msg = f"{annotation!r} is not a shape JSON takes" - raise TypeError(msg) - return check - - -def check_for(annotation: object) -> TypeCheck | None: - """The type check a field annotation implies, or None if it implies none. - - A small compiler over the shapes JSON takes, and no others: the - scalars (`int`, `float` for any number, `bool`, `str`), a `Literal` of names, arrays homogeneous or fixed, unions of - those, a nested object described by a TypedDict or a record dataclass, - an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the - type it names, and a nested metadata field -- an entity type, with or - without `Opaque`, which `is_nested_field` recognises. `UNSET` in a union - says the member may be absent, which is the other half of a table - entry and is read separately by `is_optional`. - - Closed: an annotation outside these implies no check, and an entity - declaring one is refused at class creation. The field is written as - one of these shapes instead, with any finer rule in `__post_init__`. - """ - inner, _ = strip_annotation(annotation) - if is_nested_field(inner): - return is_metadata_field - if inner is int: - return is_int - if inner is float: - return is_number - if inner is bool: - return is_bool - if inner is str: - return is_str - if inner is JSONValue: - return is_json_value - if get_origin(inner) is Literal: - return _compile_literal(inner) - if is_union(inner): - return _compile_union(inner) - if get_origin(inner) is tuple: - return _compile_tuple(inner) - if is_typeddict(inner): - return _compile_typeddict(inner) - if get_origin(inner) in (Mapping, dict): - return _compile_mapping(inner) - if isinstance(inner, NewType): - return _compile_new_type(inner) - # Last, because `is_dataclass` narrows what pyright knows of `inner` - # for every line after it. - if isinstance(inner, type) and is_dataclass(inner): - return _compile_record(inner) - return None - - -def element_annotations(inner: object, count: int) -> list[object]: - """The annotation of each element of a tuple type, one per element held.""" - arguments = get_args(inner) - if len(arguments) == 2 and arguments[1] is Ellipsis: - return [arguments[0]] * count - return list(arguments) - - -def is_class_var(annotation: object) -> bool: - """Whether an annotation says `ClassVar`. - - `from __future__ import annotations` leaves them as strings, so this - reads the text when it gets one -- the same thing `dataclasses` does, - and for the same reason: resolving the name needs a module namespace - that is not available while the class is still being built. - """ - if isinstance(annotation, str): - stripped = annotation.strip() - return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) - return annotation is ClassVar or get_origin(annotation) is ClassVar - - -def declared_class_vars(cls: type) -> dict[str, type]: - """Every class variable annotated anywhere in `cls`'s ancestry. - - Mapped to the class that annotated it, so a message can say where the - requirement comes from. Base first, so a redeclaration names the - nearest ancestor. - """ - found: dict[str, type] = {} - for ancestor in reversed(cls.__mro__): - for name, annotation in own_annotations(ancestor).items(): - if is_class_var(annotation): - found[name] = ancestor - return found - - -__all__ = [ - "FROM_NAME", - "MetadataFieldValue", - "any_of", - "check_for", - "declared_class_vars", - "describe", - "element_annotations", - "field_hints", - "fixed_tuple", - "has_shape", - "is_class_var", - "is_from_name", - "is_metadata_field_type", - "is_nested_field", - "is_optional", - "is_union", - "mapping_of", - "own_annotations", - "shape_of", - "strip_annotation", - "type_check", - "unsubscripted", -] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 978ab9cda9..a27985fe02 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -29,7 +29,6 @@ validate_array_metadata_v3 as validate_array_metadata_v3_structure, ) from zarr_metadata.v3._chain import chain_problems -from zarr_metadata.v3._compile import field_hints from zarr_metadata.v3._entity import ( ChunkGridEntity, ChunkKeyEncodingEntity, @@ -38,8 +37,7 @@ MetadataEntity, Opaque, StorageTransformerEntity, - canonicalize_nested, - contains_entity, + canonicalized, within, written, ) @@ -91,25 +89,26 @@ def problems(self) -> tuple[ValidationProblem, ...]: def canonical(self) -> ArrayDocumentV3: """This document in the simplest form that means the same thing. - Each entity in its own canonical form -- the same walk over the - fields that hold one that every entity containing entities gets - -- and the one rule that is the document's own: `dimension_names` - of nothing but nulls says what omitting the field says. A - *transformation*, asked for by `canonicalize_array_metadata_v3`; - `to_json` does not apply it. + Each entity in its own canonical form, and the one rule that is + the document's own: `dimension_names` of nothing but nulls says + what omitting the field says. A *transformation*, asked for by + `canonicalize_array_metadata_v3`; `to_json` does not apply it. """ - entities = { - name: canonicalize_nested(annotation, getattr(self, name)) - for name, annotation in field_hints(type(self)).items() - if contains_entity(annotation) - } document = dict(self.document) names = document.get("dimension_names") if isinstance(names, tuple) and all( entry is None for entry in cast("tuple[object, ...]", names) ): del document["dimension_names"] - return replace(self, document=document, **entities) + return replace( + self, + document=document, + data_type=canonicalized(self.data_type), + chunk_grid=canonicalized(self.chunk_grid), + chunk_key_encoding=canonicalized(self.chunk_key_encoding), + codecs=tuple(canonicalized(codec) for codec in self.codecs), + storage_transformers=tuple(canonicalized(entry) for entry in self.storage_transformers), + ) def to_json(self) -> dict[str, object]: """The document as it would be written: every entity in its JSON form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 317b963055..969987f5e7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -1,109 +1,66 @@ """What every metadata entity can do for itself. -A codec, data type, chunk grid or chunk key encoding is one frozen -dataclass whose fields are its schema. Everything the layer knows about -a member's type is read off the field annotations by `_compile`: which -members there are, which may be absent, how each is type-checked, and --- for a field typed as another entity -- that it is read through the -scope, written back as its own JSON, and put in canonical form by -recursing into it. Everything finer than a type -- a bound, a rule about -a member, members read together -- is the entity's own `__post_init__`, -which collects every problem it finds and raises once; `coerce` reports -those instead of raising. - -`coerce` is the reading path: raw metadata in, the entity or the -reasons it is not one out, taking a `Context` -- the entities in scope -for this reading -- which the entities that contain other entities need. -`__init_subclass__` refuses, at class creation, every way of writing an -entity that would type-check and then misbehave somewhere that will not -name the class. +A codec, data type, chunk grid, chunk key encoding or storage transformer +is one frozen dataclass whose fields are its configuration, and the +field annotations are its schema: `coerce` reads a document's +configuration against them, member by member, with `_typed_json`. A +field typed `CodecEntity | Opaque` holds another entity, read through +the scope the containing one is read in. Everything finer than a type -- +a bound, a rule about a member, members read together -- is the entity's +own `__post_init__`, which collects every problem it finds and raises +once; `coerce` reports those instead of raising. + +What an entity writes and what it simplifies to are its own too: +`to_json` is abstract, a literal of the entity's JSON type, and +`canonical` defaults to the entity itself. An entity that contains +entities writes them with `written` and canonicalizes them with +`canonicalized`, in the same two methods. Composition -- what needs the document or the codec chain -- is the -entity's to answer too, through `incoming_problems`, `shape_problems`, +entity's to answer through `incoming_problems`, `shape_problems`, `fill_value_problems`, `transition` and `grid`, each taking the part of the document it needs. The document that composes those answers is -`_document`. +`_document`; which entities are in scope is `_registry`. """ from __future__ import annotations from abc import ABC, abstractmethod - -# Runtime imports, not `TYPE_CHECKING` ones: the string type aliases below -# (`TypeCheck`, `MemberTypes`) are resolved by `get_type_hints` at class -# creation, and a name that exists only for the type checker is a NameError -# then -- for this package and for any tool introspecting an entity. -from collections.abc import Callable, Mapping # noqa: TC003 -from dataclasses import MISSING, Field, dataclass, is_dataclass, replace -from typing import ( - TYPE_CHECKING, - ClassVar, - Final, - Literal, - TypeAlias, - cast, - final, - get_args, - get_origin, - get_type_hints, -) - -from typing_extensions import TypeVar, is_typeddict +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast, get_args from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ( - MetadataValidationError, - ValidationProblem, -) - -if TYPE_CHECKING: - from typing import Self - - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - from zarr_metadata.v3._parts import ArrayParts, ChunkGrid - from zarr_metadata.v3._registry import Context - -from zarr_metadata.v3._checks import ( +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.v3._typed_json import ( Loc, - TypeCheck, + Parsed, + Parser, as_tuples, - is_bool, - is_int, - is_integer, - is_json_value, - is_metadata_field, - is_str, - named_configuration, - one_of, - problem, - sequence_of, - within, -) -from zarr_metadata.v3._compile import ( - FROM_NAME, - MetadataFieldValue, - check_for, declared_class_vars, - element_annotations, field_hints, - has_shape, - is_class_var, - is_from_name, - is_metadata_field_type, - is_nested_field, + is_integer, is_optional, is_union, - own_annotations, - shape_of, + parser, + parser_for, + problem, strip_annotation, - type_check, ) +if TYPE_CHECKING: + from collections.abc import Sequence + from typing import Self + + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._parts import ArrayParts, ChunkGrid + from zarr_metadata.v3._registry import Context + from zarr_metadata.v3._typed_json import Leaf + EntityT = TypeVar("EntityT", bound="MetadataEntity") -# A real alias, not a string one: entity modules subscript it as -# `Coerced[Self]` in a return annotation, and not all of them defer -# annotation evaluation. +# A real alias, not a string one, so a return annotation can subscript +# it -- `Coerced[Self]` -- without deferring annotation evaluation. Coerced: TypeAlias = tuple[EntityT | None, tuple[ValidationProblem, ...]] """The entity if it could be built, and every problem found. @@ -128,158 +85,90 @@ """ -CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] -"""The three pipeline positions the v3 spec sorts codecs into. +class _FromName: + """The marker behind `FROM_NAME`.""" + + __slots__ = () + + def __repr__(self) -> str: + return "FROM_NAME" + -Declared by each codec, which is why there is no table of it: a name -does not have a pipeline position, a codec does. +FROM_NAME: Final = _FromName() +"""Marks a field carried by the metadata envelope's `name`, not its configuration. + + data_type_name: Annotated[str, FROM_NAME] + +A member all the same -- `__post_init__` judges it -- but not a +configuration key, so it is neither read from nor written to a +`configuration` object. The raw-bytes family is the case: `r` keeps its +width in its name and has no configuration at all. """ -def contains_entity(annotation: object) -> bool: - """Whether a value of this type holds a nested metadata field anywhere in it.""" - inner, _ = strip_annotation(annotation) - if is_metadata_field_type(inner): - return True - origin = get_origin(inner) - if is_union(inner): - return any(contains_entity(arg) for arg in get_args(inner) if arg is not UNSET) - if origin is tuple: - return any(contains_entity(arg) for arg in get_args(inner) if arg is not Ellipsis) - if is_typeddict(inner): - return any( - contains_entity(value) for value in get_type_hints(inner, include_extras=True).values() - ) - if isinstance(inner, type) and is_dataclass(inner): - return any(contains_entity(value) for value in field_hints(inner).values()) - return False - - -def _entity_class(annotation: object) -> type[MetadataEntity] | None: - """`annotation` when it is an entity class, else None.""" - if isinstance(annotation, type) and issubclass(annotation, MetadataEntity): - return annotation - return None - - -def _entity_kinds(annotation: object) -> list[type[MetadataEntity]]: - """Every entity type an annotation names, at any depth.""" - inner, _ = strip_annotation(annotation) - kind = _entity_class(inner) - if kind is not None: - return [kind] - origin = get_origin(inner) - arguments: tuple[object, ...] = get_args(inner) - if is_union(inner): - return [kind for arg in arguments if arg is not UNSET for kind in _entity_kinds(arg)] - if origin is tuple: - return [kind for arg in arguments if arg is not Ellipsis for kind in _entity_kinds(arg)] - if isinstance(inner, type) and is_dataclass(inner) and not is_metadata_field_type(inner): - return [kind for value in field_hints(inner).values() for kind in _entity_kinds(value)] - return [] - - -def _fitting_branch(inner: object, value: object) -> object | None: - """The branch of a union that holds an entity and whose shape `value` has.""" - for branch in get_args(inner): - if branch is UNSET or not contains_entity(branch): - continue - if has_shape(shape_of(branch), value): - return branch - return None - - -def _resolve( - annotation: object, value: object, context: Context, loc: Loc -) -> tuple[object, tuple[ValidationProblem, ...]]: - """`value`, with every nested metadata field in it read as an entity in `context`. - - A field annotated with an entity type is resolved through the scope, - at the point that kind of entity is registered at; an array of them - element by element; a record holding one field by field. The value - has passed its type check, so the shapes are the annotation's. +def is_from_name(annotation: object) -> bool: + """Whether `FROM_NAME` marks the field: carried by the envelope's name, not a configuration key.""" + return any(entry is FROM_NAME for entry in strip_annotation(annotation)[1]) + + +def within(prefix: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: + """One entity's problems, located in the document that holds it. + + An entity reports relative to its own `configuration`, so that is what + goes between the field and the member. A problem with an empty + location is about the entity itself -- a malformed `r` name, a + codec that cannot encode what reaches it -- and lands on the field. """ - inner, _ = strip_annotation(annotation) - if is_nested_field(inner): - return context.coerce(_entity_kinds(inner)[0], value, loc) - if is_union(inner): - branch = _fitting_branch(inner, value) - return (value, ()) if branch is None else _resolve(branch, value, context, loc) - if get_origin(inner) is tuple: - entries = cast("tuple[object, ...]", value) - resolved: list[object] = [] - found: list[ValidationProblem] = [] - for position, (element, entry) in enumerate( - zip(element_annotations(inner, len(entries)), entries, strict=True) - ): - item, problems = _resolve(element, entry, context, (*loc, position)) - resolved.append(item) - found.extend(problems) - return tuple(resolved), tuple(found) - if isinstance(inner, type) and is_dataclass(inner) and not is_metadata_field_type(inner): - entries = cast("Mapping[str, object]", value) - members: dict[str, object] = {} - found = [] - for name, field_annotation in field_hints(inner).items(): - if name not in entries: - continue - member, problems = _resolve(field_annotation, entries[name], context, (*loc, name)) - members[name] = member - found.extend(problems) - return inner(**members), tuple(found) - return value, () + return tuple( + ValidationProblem( + (*prefix, *(("configuration", *found.loc) if len(found.loc) != 0 else ())), + found.message, + found.kind, + ) + for found in problems + ) -def written(value: MetadataEntity | Opaque) -> ZarrV3MetadataFieldJSON: - """A contained metadata field as a document would write it. +def named_configuration( + value: object, +) -> tuple[str | None, Mapping[str, object] | None, bool]: + """Split metadata into `(name, configuration, must_understand)`. - The entity's own JSON, or the JSON an `Opaque` kept. An `Opaque` - inside a built entity is out of scope -- an inner name no entity in - scope claimed -- and its JSON passed the envelope check as a metadata - field, which is what the cast says. + The shared shape every entity arrives in: a bare name, or an object + carrying one. A `None` name means the value is not a metadata field at + all; a `None` configuration means the bare spelling was used. """ - if isinstance(value, MetadataEntity): - return value.to_json() - return cast("ZarrV3MetadataFieldJSON", value.json) + if isinstance(value, str): + return value, None, True + if not isinstance(value, Mapping): + return None, None, True + entry = cast("Mapping[str, object]", value) + name = entry.get("name") + if not isinstance(name, str): + return None, None, True + configuration = entry.get("configuration") + must_understand = entry.get("must_understand", True) + return ( + name, + cast("Mapping[str, object]", configuration) if isinstance(configuration, Mapping) else None, + must_understand if isinstance(must_understand, bool) else True, + ) -def canonicalize_nested(annotation: object, value: object) -> object: - """`value` with every nested entity in its own canonical form.""" - if isinstance(value, MetadataEntity): - return value.canonical() - if isinstance(value, Opaque): - return value - inner, _ = strip_annotation(annotation) - if is_union(inner): - branch = _fitting_branch(inner, value) - return value if branch is None else canonicalize_nested(branch, value) - if get_origin(inner) is tuple: - entries = cast("tuple[object, ...]", value) - return tuple( - canonicalize_nested(element, entry) - for element, entry in zip( - element_annotations(inner, len(entries)), entries, strict=True - ) - ) - if ( - isinstance(inner, type) - and is_dataclass(inner) - and not is_metadata_field_type(inner) - and is_dataclass(value) - and not isinstance(value, type) - ): - return replace( - value, - **{ - name: canonicalize_nested(field_annotation, getattr(value, name)) - for name, field_annotation in field_hints(inner).items() - }, - ) - return value +def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: + """A nested metadata field: a bare name or a named-configuration object. + + Only the envelope's shape. Which entity the name denotes, and whether + its configuration is well formed, is settled when the containing + entity reads it in scope. + """ + if not isinstance(value, (str, Mapping)): + return problem(loc, f"expected a metadata field, got {value!r}") + return () @dataclass(frozen=True, slots=True) -class Opaque(MetadataFieldValue): +class Opaque: """A metadata field this reading did not turn into an entity. Carries the JSON the document wrote, so a reader holding a @@ -295,269 +184,86 @@ class Opaque(MetadataFieldValue): reason: Literal["out_of_scope", "invalid"] -# The invariants, each a function of the compiled class returning why it -# is refused, or None. Every one names something that type-checks cleanly -# and then goes wrong somewhere that will not name the class. - -_FINAL_ADVICE: Final[Mapping[str, str]] = { - "canonical": ( - "which is the walk into contained entities; put the entity's own rewrite " - "in `simplified`, which `canonical` calls after the walk" - ), -} -"""What to do instead, for each method the base marks `@final`.""" - - -def _members(cls: type[MetadataEntity]) -> dict[str, bool]: - """The configuration members, each with whether it is required: the fields, less the envelope's.""" - return { - name: not is_optional(annotation) - for name, annotation in field_hints(cls).items() - if not is_from_name(annotation) - } - - -def _nested(cls: type[MetadataEntity]) -> dict[str, object]: - """The fields that hold other entities, with their annotations.""" - return { - name: annotation - for name, annotation in field_hints(cls).items() - if contains_entity(annotation) - } - - -def _fields_are_json_shapes(cls: type[MetadataEntity]) -> str | None: - hints = field_hints(cls) - unread = sorted( - name - for name, annotation in hints.items() - if not is_from_name(annotation) and check_for(annotation) is None - ) - if len(unread) == 0: - return None - return ( - f"{cls.__name__}: " - f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in unread)}" - ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " - "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " - "record, Mapping[str, V], a NewType, or an entity kind with Opaque " - "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in `__post_init__`" - ) - - -def _final_methods_are_not_overridden(cls: type[MetadataEntity]) -> str | None: - # `@final` is a promise pyright checks in the author's editor; this - # is the same promise for a class created without one. - for name in vars(cls): - if getattr(getattr(MetadataEntity, name, None), "__final__", False): - return f"{cls.__name__} overrides `{name}`, {_FINAL_ADVICE.get(name, 'which is final')}" - return None - - -def _entities_are_of_a_kind(cls: type[MetadataEntity]) -> str | None: - # A scope holds entities by kind, so an entity of none could never be - # registered or resolved. - if kind_of(cls) is not None: - return None - return ( - f"{cls.__name__} subclasses MetadataEntity directly; subclass the kind of thing it is: " - "a codec kind, DataTypeEntity, ChunkGridEntity, ChunkKeyEncodingEntity or " - "StorageTransformerEntity" - ) +def written(value: MetadataEntity | Opaque) -> ZarrV3MetadataFieldJSON: + """A contained metadata field as a document would write it. + The entity's own JSON, or the JSON an `Opaque` kept. An `Opaque` + inside a built entity is out of scope -- an inner name no entity in + scope claimed -- and its JSON passed the envelope check as a metadata + field, which is what the cast says. + """ + if isinstance(value, MetadataEntity): + return value.to_json() + return cast("ZarrV3MetadataFieldJSON", value.json) -def _nested_fields_name_a_kind(cls: type[MetadataEntity]) -> str | None: - # A field typed as bare `MetadataEntity` could not be resolved through - # a scope: nothing says which kind's table to look in. - unplaced = sorted( - name - for name, annotation in _nested(cls).items() - if any(kind_of(kind) is None for kind in _entity_kinds(annotation)) - ) - if len(unplaced) == 0: - return None - return ( - f"{cls.__name__}: the entity type of {', '.join(unplaced)} is of no kind; annotate it " - "with a codec kind, DataTypeEntity, ChunkGridEntity, ChunkKeyEncodingEntity or " - "StorageTransformerEntity, or a subclass of one" - ) +def canonicalized(value: EntityT | Opaque) -> EntityT | Opaque: + """A contained metadata field in canonical form: the entity's own, or the `Opaque` as it is.""" + if isinstance(value, MetadataEntity): + return value.canonical() + return value -def _entity_unions_lacking_opaque(annotation: object) -> bool: - """Whether an entity kind appears in `annotation` without `Opaque` beside it.""" - inner, _ = strip_annotation(annotation) - if is_union(inner): - parts = [part for part in get_args(inner) if part is not UNSET] - if any(is_metadata_field_type(part) and part is not Opaque for part in parts): - return Opaque not in parts - return any(_entity_unions_lacking_opaque(part) for part in parts) - if is_metadata_field_type(inner): - return inner is not Opaque - if get_origin(inner) is tuple: - return any( - _entity_unions_lacking_opaque(part) for part in get_args(inner) if part is not Ellipsis - ) - if isinstance(inner, type) and is_dataclass(inner): - return any(_entity_unions_lacking_opaque(value) for value in field_hints(inner).values()) - return False - - -def _nested_fields_admit_opaque(cls: type[MetadataEntity]) -> str | None: - # A nested field holds an `Opaque` when the name is out of scope, so - # an annotation that excludes it lies to the type checker: reading - # `codec.inner.level` would be accepted and then raise. - lacking = sorted( - name - for name, annotation in _nested(cls).items() - if _entity_unions_lacking_opaque(annotation) - ) - if len(lacking) == 0: - return None - return ( - f"{cls.__name__}: {', '.join(lacking)} holds an entity but does not admit Opaque, " - "which is what it holds when the name is out of scope; annotate it as the entity " - "kind | Opaque" - ) +def nested_kind(annotation: object) -> type[MetadataEntity] | None: + """The kind an annotation of the form `Kind | Opaque` names; None if it names no entity. -def _fields_do_not_shadow_class_variables(cls: type[MetadataEntity]) -> str | None: - # A field of that name would go into the configuration and into the - # JSON -- while the class variable it - # shadows is what every other part of this layer reads. - annotated = declared_class_vars(cls) - shadowed = [ - name - for name, annotation in own_annotations(cls).items() - if name in annotated and annotated[name] is not cls and not is_class_var(annotation) + The one shape a field holding another entity takes, with `Opaque` + because that is what the field holds when the inner name is out of + scope, and a kind -- or a subclass of one, `GzipCodec` -- because a + scope resolves names by kind. An annotation naming an entity any + other way is a `TypeError` saying so, which class creation reports + against the field. + """ + parts = get_args(annotation) if is_union(annotation) else (annotation,) + entities = [ + part for part in parts if isinstance(part, type) and issubclass(part, MetadataEntity) ] - if len(shadowed) == 0: - return None - return ( - f"{cls.__name__} declares {', '.join(shadowed)} as a field, shadowing a class " - "variable of the same name; rename the field, or set the class variable instead" - ) - - -def _owed_class_variables_are_declared(cls: type[MetadataEntity]) -> str | None: - # A class variable annotated with no value anywhere in the ancestry - # is one the concrete entity owes: `identifier` for all of them, - # `kind` for a codec, `bounds` for an integer type. Derived rather - # than listed, so adding one to a family cannot forget to require it. - annotated = declared_class_vars(cls) - missing = sorted(name for name in annotated if not hasattr(cls, name)) - if len(missing) == 0: - return None - owed = ", ".join(f"{name} (annotated by {annotated[name].__name__})" for name in missing) - return ( - f"{cls.__name__} does not declare {owed}; set each as a class variable, " - "or pass base=True if this class exists only to be subclassed" - ) - - -def _codecs_are_of_a_kind(cls: type[MetadataEntity]) -> str | None: - # The kind is the base class, and what a kind must answer is abstract - # on it; a codec that skips the kind classes skips that. - if not issubclass(cls, CodecEntity): + if len(entities) == 0: return None - if issubclass(cls, (ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec)): - return None - return ( - f"{cls.__name__} subclasses CodecEntity directly; subclass ArrayArrayCodec, " - "ArrayBytesCodec or BytesBytesCodec, which says what the codec does to the array" + kind = entities[0] + if len(entities) == 1 and set(parts) == {kind, Opaque} and kind_of(kind) is not None: + return kind + msg = ( + "holds an entity but is not written as its kind | Opaque (CodecEntity | Opaque), " + "which is what the field holds when the inner name is out of scope; a kind is a " + "codec kind, DataTypeEntity, ChunkGridEntity, ChunkKeyEncodingEntity or " + "StorageTransformerEntity, or a subclass of one" ) + raise TypeError(msg) -def _class_variables_hold_listed_values(cls: type[MetadataEntity]) -> str | None: - # Other entities' rules read these and have nothing to say about a - # value outside the listed ones: the endian rule would fall silent. - listed: tuple[tuple[str, tuple[object, ...]], ...] = () - if issubclass(cls, CodecEntity): - listed = (("kind", get_args(CodecKind)),) - elif issubclass(cls, DataTypeEntity): - listed = (("scalar_storage", get_args(StorageClass)),) - for name, values in listed: - if hasattr(cls, name) and getattr(cls, name) not in values: - return f"{cls.__name__} sets {name} = {getattr(cls, name)!r}, which is not one of {values!r}" - return None - - -def _declared_defaults(cls: type[MetadataEntity]) -> dict[str, object]: - """Each member's declared default, or `MISSING`. +def _reading(context: Context | None, nested: list[ValidationProblem]) -> Leaf: + """How a field holding another entity is parsed: `Kind | Opaque`, read in `context`. - `@dataclass` has not run yet -- `__init_subclass__` runs first -- so - a member declared with `field(...)` is still a `Field` here and its - default has to be unwrapped. + Asked by the parser at every depth, so a struct's fields' data types + and a shard's inner pipelines are read the same way. The envelope's + shape is the containing field's own problem; what is found inside the + entity is collected in `nested`, apart, because the containing + entity's rules still run over its own members when only a contained + entity is wrong. With no `context` the shape is checked and nothing + is read, which is what class creation asks. """ - defaulted: dict[str, object] = {} - for key in _members(cls): - declared: object = getattr(cls, key, MISSING) - if type(declared) is Field: - spec = cast("Field[object]", declared) - declared = ( - MISSING - if spec.default is MISSING and spec.default_factory is MISSING - else spec.default - ) - defaulted[key] = declared - return defaulted - - -def _optional_members_default_to_unset(cls: type[MetadataEntity]) -> str | None: - # Or `configuration` emits the member for every instance, so the - # bare-name spelling becomes unreachable and a document gains a - # member it never wrote. - defaulted = _declared_defaults(cls) - invented = [ - key - for key, required in _members(cls).items() - if not required and defaulted[key] is not UNSET - ] - if len(invented) == 0: - return None - return ( - f"{cls.__name__} gives the optional member(s) {', '.join(invented)} a default " - "other than UNSET; write `| UNSET = UNSET` and read the meaning of absence where " - "the member is used -- a default written into every document is not a member the " - "document left out" - ) + def leaf(annotation: object) -> Parser | None: + kind = nested_kind(annotation) + if kind is None: + return None -def _required_members_have_no_default(cls: type[MetadataEntity]) -> str | None: - # A required member with a default is an entity that can be built - # without it -- and then serializes a document nobody wrote. A - # conventional starting point is a `create_default` classmethod, - # named so that asking for one is deliberate. - defaulted = _declared_defaults(cls) - presumed = [ - key for key, required in _members(cls).items() if required and defaulted[key] is not MISSING - ] - if len(presumed) == 0: - return None - return ( - f"{cls.__name__} gives the required member(s) {', '.join(presumed)} a default; " - "either drop the default, or make the member optional with `| UNSET = UNSET`" - ) + def parse(value: object, loc: Loc) -> Parsed: + problems = is_metadata_field(value, loc) + if len(problems) != 0 or context is None: + return value, problems + entity, found = context.coerce(kind, value, loc) + nested.extend(found) + return entity, () + return parse -_INVARIANTS: Final[tuple[Callable[[type[MetadataEntity]], str | None], ...]] = ( - _fields_are_json_shapes, - _final_methods_are_not_overridden, - _entities_are_of_a_kind, - _nested_fields_name_a_kind, - _nested_fields_admit_opaque, - _fields_do_not_shadow_class_variables, - _owed_class_variables_are_declared, - _codecs_are_of_a_kind, - _class_variables_hold_listed_values, - _optional_members_default_to_unset, - _required_members_have_no_default, -) -"""What a compiled entity must satisfy, asked in this order at class creation.""" + return leaf @dataclass(frozen=True) -class MetadataEntity(MetadataFieldValue, ABC): +class MetadataEntity(ABC): """One named entity, coerced from its metadata. Subclasses add their configuration members as fields, which is what @@ -575,13 +281,13 @@ class MetadataEntity(MetadataFieldValue, ABC): mapping instead: `MappingProxyType` is unhashable too, and anything else stops `json.dumps` from serializing what `to_json` returns. - A subclass writes its fields and, where the spec has something to - say beyond their types, a `__post_init__` that collects every problem - and raises `MetadataValidationError` once -- so `BloscCodec(clevel=99)` - raises, and `coerce` reports the same problems instead. `coerce`, - `coerce` and `canonical` are written once here against what the - fields say, read off them as needed; `to_json` is the entity's own, - a literal of its JSON type. + A subclass writes its fields; a `__post_init__` where the spec has + something to say beyond their types, collecting every problem and + raising `MetadataValidationError` once -- so `BloscCodec(clevel=99)` + raises, and `coerce` reports the same problems instead; `to_json`, a + literal of its JSON type; and `canonical` where two spellings of its + members mean the same. `coerce` is written once here, against what + the fields say. """ identifier: ClassVar[str] @@ -593,13 +299,15 @@ class MetadataEntity(MetadataFieldValue, ABC): """ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: - """Compile the entity from its fields, and refuse one this layer cannot use. + """Refuse, at class creation, an entity this layer could not read. - Every invariant in `_INVARIANTS` is asked, the first of which - refuses a field annotation outside the shapes the compiler reads. Each names - something that type-checks cleanly and then goes wrong later, - somewhere that will not name this class; an import-time error in - the extension's own module is the one place the author is looking. + Two things type-check cleanly and then go wrong somewhere that + will not name the class: a field whose annotation is not a shape + JSON takes, which `coerce` could not parse, and a class variable + a base annotates and nothing sets -- `identifier` for every + entity, `bounds` for an integer type -- which the first lookup + would fail. An import-time error in the extension's own module is + the one place the author is looking. `base=True` for a class that exists to add a class variable rather than to be an entity -- `CodecEntity`, `IntegerDataType`. @@ -607,15 +315,38 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: super().__init_subclass__(**kwargs) if base: return - if "__dataclass_fields__" in vars(cls): - # `@dataclass(slots=True)` builds the class a second time from - # the first one's dict: verified already, and its members are - # slot descriptors now rather than the defaults the checks read. - return - for invariant in _INVARIANTS: - message = invariant(cls) - if message is not None: - raise TypeError(message) + hints = field_hints(cls) + unread: list[str] = [] + for name, annotation in hints.items(): + try: + accepted = parser_for(annotation, _reading(None, [])) is not None + except TypeError as refused: + msg = f"{cls.__name__}: {name} {refused}" + raise TypeError(msg) from None + if not accepted: + unread.append(name) + if len(unread) != 0: + msg = ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in unread)}" + ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in `__post_init__`" + ) + raise TypeError(msg) + annotated = declared_class_vars(cls) + missing = sorted(name for name in annotated if not hasattr(cls, name)) + if len(missing) != 0: + owed = ", ".join( + f"{name} (annotated by {annotated[name].__name__})" for name in missing + ) + msg = ( + f"{cls.__name__} does not declare {owed}; set each as a class variable, " + "or pass base=True if this class exists only to be subclassed" + ) + raise TypeError(msg) @classmethod def accepts(cls, name: str) -> bool: @@ -630,83 +361,78 @@ def accepts(cls, name: str) -> bool: def coerce(cls, value: object, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one. - `context` is the scope this reading is happening in; most entities - have no use for it and ignore it. + Each configuration member is parsed against its field's + annotation; a member holding another entity is read in `context`, + the scope this reading is happening in. An optional member the + document left out is passed as `UNSET`, so no field's default + decides what a document said. The entity is built only when every + member of its own read -- its rules are written over a whole + configuration -- and handed back only when everything inside it + read too. """ - name, configuration, _ = named_configuration(value) + name, given, _ = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), f"expected the {cls.identifier!r} entity") hints = field_hints(cls) - if configuration is None: - if any(_members(cls).values()): - return None, problem( - ("configuration",), - f"{cls.identifier!r} requires a configuration", - "missing_key", - ) - configuration = cast("Mapping[str, object]", {}) + if given is None and any( + not is_from_name(annotation) and not is_optional(annotation) + for annotation in hints.values() + ): + return None, problem( + ("configuration",), + f"{cls.identifier!r} requires a configuration", + "missing_key", + ) + configuration: Mapping[str, object] = {} if given is None else given + nested: list[ValidationProblem] = [] + reading = _reading(context, nested) members: dict[str, object] = {} - found: list[ValidationProblem] = [] + own: list[ValidationProblem] = [] for key in configuration: if key not in hints or is_from_name(hints[key]): - found.extend( + own.extend( problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") ) for key, annotation in hints.items(): if is_from_name(annotation): members[key] = name - continue - if key not in configuration: - if not is_optional(annotation): - found.extend( + elif key not in configuration: + if is_optional(annotation): + members[key] = UNSET + else: + own.extend( problem( ("configuration", key), f"missing required key {key!r}", "missing_key" ) ) - continue - # Normalized before the check, so a check only ever sees the - # tuples the TypedDicts declare, never the lists raw JSON - # arrives as. A member of the wrong type is reported and left - # out; an unknown key inside it is survivable. - member = as_tuples(configuration[key]) - problems = type_check(annotation)(member, ("configuration", key)) - found.extend(problems) - if all(entry.kind == "unknown_key" for entry in problems): - members[key] = member - own = tuple(found) - # A member that is itself an entity is read in the scope whatever - # else was found: its problems are determinable, so they are - # reported in the same pass. - for key, annotation in hints.items(): - if key in members and contains_entity(annotation): - members[key], nested = _resolve( - annotation, members[key], context, ("configuration", key) + else: + # Arrays as tuples before parsing, so a member holds the + # tuples its type declares, never the lists raw JSON + # arrives as. + members[key], problems = parser(annotation, reading)( + as_tuples(configuration[key]), ("configuration", key) ) - found.extend(nested) - found_all = tuple(found) + own.extend(problems) + found = (*own, *nested) if any(entry.kind != "unknown_key" for entry in own): - # One of this entity's own members could not be read. That - # leaves a hole, and the rules are written over a whole - # configuration -- blosc's `typesize` requirement reads - # `shuffle` -- so judging around it would be guessing: the - # entity is not built, and the type problems stand alone. - return None, found_all + # An unknown key is survivable; a member that could not be + # read is a hole, and judging around it would be guessing. + return None, found try: entity = cls(**members) except MetadataValidationError as refused: # `__post_init__` found values the spec disallows: reported # rather than raised, located under the configuration. - return None, (*found_all, *within((), refused.problems)) - if any(entry.kind != "unknown_key" for entry in found_all): + return None, (*found, *within((), refused.problems)) + if any(entry.kind != "unknown_key" for entry in nested): # A contained entity could not be read. This entity's own # rules ran -- an invalid inner is an `Opaque`, as an # out-of-scope one is -- but what is handed back is not an # entity that would be asked composition questions it cannot # answer. - return None, found_all - return entity, found_all + return None, found + return entity, found - @final def canonical(self) -> Self: """This entity in the simplest form that means the same thing. @@ -716,26 +442,13 @@ def canonical(self) -> Self: a reader that reads and writes should not change bytes it was not asked to change. - Two steps, each with one owner. Every contained entity is put in - its own canonical form by walking the fields that hold one, which - is read off the annotations and is this method's alone -- an - override of it is refused at class creation. Then `simplified`, - the entity's own rewrite, which is where an entity says that two - spellings of its own members mean the same. + The default is the entity itself. Override it where two spellings + of the entity's members mean the same -- a rectilinear + dimension's run-length encoding, a `typesize` that `noshuffle` + ignores -- and, in an entity that contains entities, to put those + in canonical form: `replace(self, inner=canonicalized(self.inner))`. """ - nested = _nested(type(self)) - walked = ( - self - if len(nested) == 0 - else replace( - self, - **{ - name: canonicalize_nested(annotation, getattr(self, name)) - for name, annotation in nested.items() - }, - ) - ) - return walked.simplified() + return self @abstractmethod def to_json(self) -> ZarrV3MetadataFieldJSON: @@ -757,19 +470,6 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: `written`. """ - def simplified(self) -> Self: - """This entity with its own members in their simplest equivalent spelling. - - The hook `canonical` calls once every contained entity is in - canonical form. Override it where two spellings of the entity's - *own* members mean the same -- a rectilinear dimension's - run-length encoding, a `typesize` that `noshuffle` ignores -- and - return the entity rewritten. The default is the identity, and - there is nothing to call `super()` for: the walk into contained - entities is not this method's to keep. - """ - return self - @dataclass(frozen=True) class CodecEntity(MetadataEntity, base=True): @@ -780,9 +480,6 @@ class CodecEntity(MetadataEntity, base=True): pipeline the codec may stand, and what it must answer. """ - kind: ClassVar[CodecKind] - """Set by the kind class.""" - variable_size: ClassVar[bool] = False """Whether this codec's output size depends on the bytes it is given. @@ -805,8 +502,6 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb class ArrayArrayCodec(CodecEntity, base=True): """A codec that transforms the array: what reaches the next codec is its to say.""" - kind: ClassVar[CodecKind] = "array_array" - @abstractmethod def transition(self, incoming: ArrayParts) -> ArrayParts | None: """What the next codec in the chain sees. @@ -822,15 +517,11 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: class ArrayBytesCodec(CodecEntity, base=True): """The one codec in a pipeline that turns the array into bytes.""" - kind: ClassVar[CodecKind] = "array_bytes" - @dataclass(frozen=True) class BytesBytesCodec(CodecEntity, base=True): """A codec that transforms bytes, after the array is gone.""" - kind: ClassVar[CodecKind] = "bytes_bytes" - @dataclass(frozen=True) class ChunkGridEntity(MetadataEntity, base=True): @@ -917,7 +608,6 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "ChunkGridEntity", "ChunkKeyEncodingEntity", "CodecEntity", - "CodecKind", "Coerced", "DataTypeEntity", "Loc", @@ -925,18 +615,14 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "Opaque", "StorageClass", "StorageTransformerEntity", - "TypeCheck", - "is_bool", - "is_int", + "canonicalized", + "is_from_name", "is_integer", - "is_json_value", "is_metadata_field", - "is_str", "kind_of", "named_configuration", - "one_of", + "nested_kind", "problem", - "sequence_of", "within", "written", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 2caf1226a7..5af57deb12 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -32,14 +32,18 @@ ValidationProblem, validate_metadata_field_v3, ) -from zarr_metadata.v3._compile import is_class_var, own_annotations from zarr_metadata.v3._entity import ( KINDS, + ArrayArrayCodec, + ArrayBytesCodec, + BytesBytesCodec, + CodecEntity, MetadataEntity, Opaque, kind_of, named_configuration, ) +from zarr_metadata.v3._typed_json import is_class_var, own_annotations from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid from zarr_metadata.v3.chunk_grid.regular import RegularChunkGrid from zarr_metadata.v3.chunk_key_encoding.default import DefaultChunkKeyEncoding @@ -74,10 +78,6 @@ from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType -if TYPE_CHECKING: - from zarr_metadata.v3._entity import Loc - - if TYPE_CHECKING: from zarr_metadata.v3._entity import Loc @@ -216,8 +216,9 @@ def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: """The kind `entity` is registered under; `TypeError` for a class no scope can use. Class creation refuses what it can see; these are the things it - cannot -- the decorator, what a kind leaves abstract -- checked at - the first place the class passes through before `coerce` builds it. + cannot -- the decorator, what a kind leaves abstract, which base was + chosen -- checked at the first place the class passes through before + `coerce` builds it. """ kind = kind_of(entity) if kind is None: @@ -226,6 +227,16 @@ def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: "ChunkGridEntity, ChunkKeyEncodingEntity or StorageTransformerEntity" ) raise TypeError(msg) + if issubclass(entity, CodecEntity) and not issubclass( + entity, (ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec) + ): + # The kind classes say what a codec does to the array, and what + # each must answer is abstract on them. + msg = ( + f"{entity.__name__} subclasses CodecEntity directly; subclass ArrayArrayCodec, " + "ArrayBytesCodec or BytesBytesCodec, which says what the codec does to the array" + ) + raise TypeError(msg) if inspect.isabstract(entity): left = ", ".join(sorted(entity.__abstractmethods__)) msg = f"{entity.__name__} does not define {left}, which its base leaves abstract" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py new file mode 100644 index 0000000000..04715bb847 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -0,0 +1,649 @@ +"""JSON values parsed by type annotation. + +A dataclass's fields are its schema, and this module reads that schema. +`parser_for` turns a field annotation into a parser: a function of a +JSON value and its location that returns the typed value and every +problem found with it. The annotations it reads are the shapes JSON +takes and no others -- `int`, `float` for any number, `bool`, `str`, +`JSONValue`, a `Literal` of names, `tuple[T, ...]` and `tuple[T1, T2]`, +a union of those, an object described by a TypedDict or a record +dataclass, `Mapping[str, V]`, a `NewType` as the type it names -- which +is what keeps it small. `UNSET` in a union says the member may be +absent; `is_optional` reads that, and a parser only ever sees a present +value. + +Nothing here knows what a metadata entity is. A caller with a shape of +its own -- a field that holds another entity, read through a scope -- +passes a `leaf`, which is asked first for every annotation at every +depth; the parser it returns is used as it is. +""" + +from __future__ import annotations + +import sys +import types +from collections.abc import Callable, Mapping, Sequence +from dataclasses import is_dataclass +from typing import ( + TYPE_CHECKING, + Annotated, + ClassVar, + Literal, + NewType, + NotRequired, + Required, + TypeAlias, + Union, + cast, + get_args, + get_origin, + get_type_hints, +) + +from typing_extensions import ReadOnly, TypeIs, is_typeddict + +from zarr_metadata._common import JSONValue +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ValidationProblem, is_json + +if TYPE_CHECKING: + from zarr_metadata.model._validation import ProblemKind + + +Loc: TypeAlias = "tuple[str | int, ...]" +"""Where in a document a value sits: the keys and indices down to it.""" + +Parsed: TypeAlias = "tuple[object, tuple[ValidationProblem, ...]]" +"""What a parser returns: the typed value, and every problem found with it.""" + +Parser: TypeAlias = "Callable[[object, Loc], Parsed]" +"""One value against one annotation, located at `loc`.""" + +Leaf: TypeAlias = "Callable[[object], Parser | None]" +"""A caller's own shapes: asked first for every annotation, None to decline.""" + + +def problem( + loc: Loc, message: str, kind: ProblemKind = "invalid_type" +) -> tuple[ValidationProblem, ...]: + """One problem, as the one-element tuple every parser returns. + + A tuple so that a parser can return it directly and a rule can + `found.extend(problem(...))` and raise `MetadataValidationError(found)` + once. The default `kind` names a type mismatch; a value rule passes + `"invalid_value"`. + """ + return (ValidationProblem(loc, message, kind),) + + +def is_integer(value: object) -> TypeIs[int]: + """A JSON integer: an `int`, and not a `bool`. + + `True` is an `int` in Python and `true` is not a number in JSON, so + the two have to be told apart everywhere a number is expected. + """ + return not isinstance(value, bool) and isinstance(value, int) + + +def as_tuples(value: object) -> object: + """Every JSON array in `value`, at any depth, as a tuple. + + The TypedDicts spell a JSON array as a tuple throughout, so a member + taken straight from parsed JSON would otherwise hold a list where its + own type says tuple -- and two documents differing only in that would + compare unequal. + """ + if isinstance(value, (list, tuple)): + entries = cast("list[object] | tuple[object, ...]", value) + return tuple(as_tuples(entry) for entry in entries) + if isinstance(value, Mapping): + entries = cast("Mapping[str, object]", value) + return {key: as_tuples(entry) for key, entry in entries.items()} + return value + + +# --- annotations --------------------------------------------------------- + + +def strip_annotation(annotation: object) -> tuple[object, tuple[object, ...]]: + """An annotation's type, and the metadata `Annotated` wrapped it in. + + `NotRequired`, `Required` and `ReadOnly` are qualifiers on a TypedDict + key, not part of the value's type; peeled with the `Annotated` layers, + in whatever order they were written. + """ + metadata: list[object] = [] + while True: + origin = get_origin(annotation) + if origin is Annotated: + inner, *extras = get_args(annotation) + metadata.extend(extras) + annotation = inner + elif origin in (NotRequired, Required, ReadOnly): + (annotation,) = get_args(annotation) + else: + return annotation, tuple(metadata) + + +def is_not_required(annotation: object) -> bool: + """Whether a TypedDict key is marked `NotRequired`, under whatever qualifiers.""" + while True: + origin = get_origin(annotation) + if origin is NotRequired: + return True + if origin is Annotated: + annotation = get_args(annotation)[0] + elif origin in (Required, ReadOnly): + (annotation,) = get_args(annotation) + else: + return False + + +def is_union(annotation: object) -> bool: + return get_origin(annotation) in (Union, types.UnionType) + + +def is_optional(annotation: object) -> bool: + """Whether a field may be absent: its type admits `UNSET`.""" + inner, _ = strip_annotation(annotation) + return is_union(inner) and any(arg is UNSET for arg in get_args(inner)) + + +def without_unset(inner: object) -> object: + """The type of a present value: the union less `UNSET`.""" + if not is_union(inner): + return inner + arguments = get_args(inner) + present = [argument for argument in arguments if argument is not UNSET] + if len(present) == len(arguments): + return inner + if len(present) == 1: + return present[0] + return Union[tuple(present)] # noqa: UP007 - built from a tuple, which `|` cannot be + + +def own_annotations(klass: type) -> dict[str, object]: + """A class's own annotations, unevaluated. + + From 3.14 a class does not carry an `__annotations__` dict until it is + asked for one, and asking evaluates every annotation at once -- so a + `ClassVar` naming something imported only for the type checker would + fail the whole class. `annotationlib` can hand them back as the text + they were written as, which is what the callers here want anyway: + class variables are skipped by text before anything is evaluated. + Earlier versions leave the dict on the class, strings or values as + the module chose. + """ + if sys.version_info >= (3, 14): + import annotationlib + + return dict(annotationlib.get_annotations(klass, format=annotationlib.Format.STRING)) + return dict(vars(klass).get("__annotations__", {})) + + +def is_class_var(annotation: object) -> bool: + """Whether an annotation says `ClassVar`. + + `from __future__ import annotations` leaves them as strings, so this + reads the text when it gets one -- the same thing `dataclasses` does, + and for the same reason: resolving the name needs a module namespace + that is not available while the class is still being built. + """ + if isinstance(annotation, str): + stripped = annotation.strip() + return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) + return annotation is ClassVar or get_origin(annotation) is ClassVar + + +def field_hints(cls: type) -> dict[str, object]: + """The dataclass fields of `cls`, resolved, base first. + + Each class's own annotations are resolved in that class's module, + and class variables are skipped *before* resolving, by text -- so a + `ClassVar` whose annotation names something imported only for the + type checker cannot fail class creation. `@dataclass` sees the same + set, in the same order. + """ + hints: dict[str, object] = {} + for ancestor in reversed(cls.__mro__): + raw = { + name: annotation + for name, annotation in own_annotations(ancestor).items() + if not is_class_var(annotation) + } + if len(raw) == 0: + continue + shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) + hints.update(get_type_hints(shell, include_extras=True)) + return hints + + +def declared_class_vars(cls: type) -> dict[str, type]: + """Every class variable annotated anywhere in `cls`'s ancestry. + + Mapped to the class that annotated it, so a message can say where the + requirement comes from. Base first, so a redeclaration names the + nearest ancestor. + """ + found: dict[str, type] = {} + for ancestor in reversed(cls.__mro__): + for name, annotation in own_annotations(ancestor).items(): + if is_class_var(annotation): + found[name] = ancestor + return found + + +# --- what a message calls a shape, and which shape a value has ------------- + + +def describe(annotation: object) -> str: + """The annotation as a message would name it: "an integer", "an object".""" + inner = without_unset(strip_annotation(annotation)[0]) + if inner is int: + return "an integer" + if inner is float: + return "a number" + if inner is bool: + return "a boolean" + if inner is str: + return "a string" + if inner is JSONValue: + return "a JSON value" + origin = get_origin(inner) + if origin is Literal: + return f"one of {tuple(sorted(get_args(inner)))!r}" + if is_union(inner): + return " or ".join(describe(branch) for branch in get_args(inner)) + if origin is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + return f"an array of {describe(arguments[0])} elements" + if len(arguments) == 2: + return f"a [{describe(arguments[0])}, {describe(arguments[1])}] pair" + return f"an array of {len(arguments)} elements" + if origin in (Mapping, dict): + return "an object" + if isinstance(inner, NewType): + return describe(inner.__supertype__) + if is_typeddict(inner) or is_dataclass(inner): + return "an object" + return "a value" + + +def shape_of(annotation: object) -> str | None: + """The top-level JSON shape an annotation admits, for choosing a union branch. + + None means any shape -- a JSON value, a union that mixes them, or a + shape the caller's `leaf` reads. + """ + inner = without_unset(strip_annotation(annotation)[0]) + if inner is int: + return "int" + if inner is float: + return "number" + if inner is bool: + return "bool" + if inner is str: + return "str" + origin = get_origin(inner) + if origin is Literal: + values = get_args(inner) + return "int" if all(isinstance(value, int) for value in values) else "str" + if origin is tuple: + return "tuple" + if origin in (Mapping, dict): + return "mapping" + if isinstance(inner, NewType): + return shape_of(inner.__supertype__) + if is_typeddict(inner) or is_dataclass(inner): + return "mapping" + return None + + +def has_shape(shape: str | None, value: object) -> bool: + if shape is None: + return True + if shape == "int": + return is_integer(value) + if shape == "number": + return not isinstance(value, bool) and isinstance(value, (int, float)) + if shape == "bool": + return isinstance(value, bool) + if shape == "str": + return isinstance(value, str) + if shape == "tuple": + return isinstance(value, tuple) + return isinstance(value, Mapping) # "mapping" + + +# --- the parsers --------------------------------------------------------- + + +def _scalar(description: str, admits: Callable[[object], bool]) -> Parser: + def parse(value: object, loc: Loc) -> Parsed: + if admits(value): + return value, () + return value, problem(loc, f"expected {description}, got {value!r}") + + return parse + + +_INTEGER: Parser = _scalar("an integer", is_integer) +_NUMBER: Parser = _scalar( + "a number", lambda value: not isinstance(value, bool) and isinstance(value, (int, float)) +) +_BOOLEAN: Parser = _scalar("a boolean", lambda value: isinstance(value, bool)) +_STRING: Parser = _scalar("a string", lambda value: isinstance(value, str)) +_JSON: Parser = _scalar("a JSON value", is_json) + + +def one_of(allowed: tuple[str, ...]) -> Parser: + """A member whose type is a closed set of names.""" + + def parse(value: object, loc: Loc) -> Parsed: + if value not in allowed: + return value, problem( + loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value" + ) + return value, () + + return parse + + +def sequence_of(element: Parser) -> Parser: + """A member whose type is an array of one element type, parsed element by element.""" + + def parse(value: object, loc: Loc) -> Parsed: + if not isinstance(value, (list, tuple)): + return value, problem(loc, f"expected a sequence, got {value!r}") + entries = cast("list[object] | tuple[object, ...]", value) + parsed: list[object] = [] + found: list[ValidationProblem] = [] + for index, entry in enumerate(entries): + item, problems = element(entry, (*loc, index)) + parsed.append(item) + found.extend(problems) + return tuple(parsed), tuple(found) + + return parse + + +def fixed_tuple(elements: Sequence[Parser], description: str) -> Parser: + """A member whose type is an array of a fixed length, parsed position by position.""" + + def parse(value: object, loc: Loc) -> Parsed: + if not isinstance(value, tuple): + return value, problem(loc, f"expected {description}, got {value!r}") + entries = cast("tuple[object, ...]", value) + if len(entries) != len(elements): + return entries, problem(loc, f"expected {description}, got {entries!r}") + parsed: list[object] = [] + found: list[ValidationProblem] = [] + for position, (element, entry) in enumerate(zip(elements, entries, strict=True)): + item, problems = element(entry, (*loc, position)) + parsed.append(item) + found.extend(problems) + return tuple(parsed), tuple(found) + + return parse + + +def any_of(branches: Sequence[tuple[object, Parser]], description: str) -> Parser: + """A member whose type is a union of shapes, parsed by the branch it fits. + + The branch whose top-level shape the value has is the one that + reports -- so an element inside a malformed array is located inside + the array, rather than the whole array being called wrong. A value + fitting no branch's shape is reported once, by what was expected; + one fitting several is parsed by the first that accepts it, and + reported by the first that does not. + """ + + def parse(value: object, loc: Loc) -> Parsed: + first: Parsed | None = None + for annotation, branch in branches: + if not has_shape(shape_of(annotation), value): + continue + result = branch(value, loc) + if len(result[1]) == 0: + return result + if first is None: + first = result + if first is None: + return value, problem(loc, f"expected {description}, got {value!r}") + return first + + return parse + + +Members: TypeAlias = "Mapping[str, tuple[bool, Parser]]" +"""An object's declared keys: whether each is required, and its parser.""" + + +def _keys( + members: Members, entries: Mapping[str, object], loc: Loc +) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: + """The declared keys of one object, each parsed at its own key. + + Closed, like every configuration in this package: a key the type does + not declare is `unknown_key`, a required one missing is `missing_key`, + both located at the object. An optional key left out is parsed as + `UNSET`, so a record never depends on a default for it. + """ + parsed: dict[str, object] = {} + found: list[ValidationProblem] = [] + for key in entries: + if key not in members: + found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) + for key, (required, member) in members.items(): + if key not in entries: + if required: + found.extend(problem(loc, f"missing required key {key!r}", "missing_key")) + else: + parsed[key] = UNSET + continue + item, problems = member(entries[key], (*loc, key)) + parsed[key] = item + found.extend(problems) + return parsed, tuple(found) + + +def object_of(members: Members) -> Parser: + """A member that is itself an object with declared keys, kept as the mapping it came as. + + A key the type does not declare is reported and kept: the member + still says what the document said. + """ + + def parse(value: object, loc: Loc) -> Parsed: + if not isinstance(value, Mapping): + return value, problem(loc, f"expected an object, got {value!r}") + entries = cast("Mapping[str, object]", value) + parsed, found = _keys(members, entries, loc) + return {**entries, **{key: item for key, item in parsed.items() if key in entries}}, found + + return parse + + +def record_of(record: Callable[..., object], members: Members) -> Parser: + """A member that is itself an object with declared keys, built as a dataclass. + + Built only from an object whose every key read; otherwise the value + comes back as it came, with the reasons. + """ + + def parse(value: object, loc: Loc) -> Parsed: + if not isinstance(value, Mapping): + return value, problem(loc, f"expected an object, got {value!r}") + entries = cast("Mapping[str, object]", value) + parsed, found = _keys(members, entries, loc) + if any(entry.kind != "unknown_key" for entry in found): + return entries, found + return record(**parsed), found + + return parse + + +def mapping_of(value: Parser) -> Parser: + """A member whose type is an object with any keys, parsed value by value. + + The open counterpart of `object_of`: a `Mapping[str, V]` says nothing + about which keys there are, only what each value must be. + """ + + def parse(candidate: object, loc: Loc) -> Parsed: + if not isinstance(candidate, Mapping): + return candidate, problem(loc, f"expected an object, got {candidate!r}") + entries = cast("Mapping[str, object]", candidate) + parsed: dict[str, object] = {} + found: list[ValidationProblem] = [] + for key, entry in entries.items(): + item, problems = value(entry, (*loc, key)) + parsed[key] = item + found.extend(problems) + return parsed, tuple(found) + + return parse + + +# --- the compiler -------------------------------------------------------- + + +def _members_of(annotations: Mapping[str, object], leaf: Leaf) -> Members | None: + """A member table for an object's keys; None if any key's type has no parser.""" + members: dict[str, tuple[bool, Parser]] = {} + for key, annotation in annotations.items(): + parser = parser_for(annotation, leaf) + if parser is None: + return None + required = not is_not_required(annotation) and not is_optional(annotation) + members[key] = (required, parser) + return members + + +def _union(inner: object, leaf: Leaf) -> Parser | None: + compiled = [(branch, parser_for(branch, leaf)) for branch in get_args(inner)] + branches = [(branch, parser) for branch, parser in compiled if parser is not None] + if len(branches) != len(compiled): + return None + return any_of(branches, describe(inner)) + + +def _tuple(inner: object, leaf: Leaf) -> Parser | None: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = parser_for(arguments[0], leaf) + return None if element is None else sequence_of(element) + compiled = [parser_for(argument, leaf) for argument in arguments] + elements = [element for element in compiled if element is not None] + if len(elements) != len(compiled): + return None + return fixed_tuple(elements, describe(inner)) + + +def _mapping(inner: object, leaf: Leaf) -> Parser | None: + arguments = get_args(inner) + if len(arguments) != 2 or arguments[0] is not str: + return None + value = parser_for(arguments[1], leaf) + return None if value is None else mapping_of(value) + + +def _no_leaf(annotation: object) -> Parser | None: + return None + + +def parser_for(annotation: object, leaf: Leaf = _no_leaf) -> Parser | None: + """The parser a field annotation implies, or None if it implies none. + + A small compiler over the shapes JSON takes and no others, listed in + the module docstring. `leaf` is asked first, here and at every depth + -- inside a union, an array, an object -- and what it returns is + used as it is. Closed: an annotation outside these implies no + parser, and a dataclass declaring one is refused at class creation. + The field is written as one of these shapes instead, with any finer + rule in `__post_init__`. + """ + inner = without_unset(strip_annotation(annotation)[0]) + found = leaf(inner) + if found is not None: + return found + if inner is int: + return _INTEGER + if inner is float: + return _NUMBER + if inner is bool: + return _BOOLEAN + if inner is str: + return _STRING + if inner is JSONValue: + return _JSON + if get_origin(inner) is Literal: + # Sorted, because the order `get_args` reports is not the order + # the `Literal` was written in: two `Literal`s over the same + # values compare and hash equal, so the first one built anywhere + # in the process is the one every later one resolves to. The + # parse is a membership test either way; this is so the message + # listing the values does not depend on import order. + return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) + if is_union(inner): + return _union(inner, leaf) + if get_origin(inner) is tuple: + return _tuple(inner, leaf) + if is_typeddict(inner): + members = _members_of(get_type_hints(inner, include_extras=True), leaf) + return None if members is None else object_of(members) + if get_origin(inner) in (Mapping, dict): + return _mapping(inner, leaf) + if isinstance(inner, NewType): + # A `NewType` is its supertype to a document; the distinction is + # the code's, for a value it has vouched for. + return parser_for(inner.__supertype__, leaf) + # Last, because `is_dataclass` narrows what pyright knows of `inner` + # for every line after it. + if isinstance(inner, type) and is_dataclass(inner): + members = _members_of(field_hints(inner), leaf) + return None if members is None else record_of(inner, members) + return None + + +def parser(annotation: object, leaf: Leaf = _no_leaf) -> Parser: + """The parser a field annotation implies; `TypeError` if it implies none.""" + found = parser_for(annotation, leaf) + if found is None: + msg = f"{annotation!r} is not a shape JSON takes" + raise TypeError(msg) + return found + + +__all__ = [ + "Leaf", + "Loc", + "Members", + "Parsed", + "Parser", + "any_of", + "as_tuples", + "declared_class_vars", + "describe", + "field_hints", + "fixed_tuple", + "has_shape", + "is_class_var", + "is_integer", + "is_not_required", + "is_optional", + "is_union", + "mapping_of", + "object_of", + "one_of", + "own_annotations", + "parser", + "parser_for", + "problem", + "record_of", + "sequence_of", + "shape_of", + "strip_annotation", + "without_unset", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 261f934b8d..024832a498 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -235,7 +235,7 @@ def grid(self, array_shape: object) -> ChunkGrid: """ return ChunkGrid.derived(tuple(_axis_lengths(spec) for spec in self.chunk_shapes)) - def simplified(self) -> Self: + def canonical(self) -> Self: """Run-length encoded, which is the spelling that does not grow. Two dimension specs listing the same extents describe the same diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index 05b484eca8..b3c4553daf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -14,13 +14,13 @@ `codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. -Each codec declares its own pipeline kind (`array -> array`, -`array -> bytes`, `bytes -> bytes`) as a `kind` class variable. +Each codec's pipeline position (`array -> array`, `array -> bytes`, +`bytes -> bytes`) is the kind class its entity subclasses, in +`zarr_metadata.v3.entity`. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ -from zarr_metadata.v3._entity import CodecKind from zarr_metadata.v3.codec.blosc import BloscCodecMetadata from zarr_metadata.v3.codec.bytes import BytesCodecMetadata from zarr_metadata.v3.codec.cast_value import CastValueCodecMetadata @@ -35,7 +35,6 @@ "BloscCodecMetadata", "BytesCodecMetadata", "CastValueCodecMetadata", - "CodecKind", "Crc32cCodecMetadata", "GzipCodecMetadata", "ScaleOffsetCodecMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index b03f9e8994..049c4a89d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -112,7 +112,7 @@ def __post_init__(self) -> None: """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. Under `noshuffle` the spec says of `typesize` that "the value is - ignored", and `simplified` drops it; under either shuffle it is + ignored", and `canonical` drops it; under either shuffle it is required, and positive. """ found: list[ValidationProblem] = [] @@ -152,7 +152,7 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) - def simplified(self) -> Self: + def canonical(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. The spec says of that case that "the value is ignored", so two diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index efbb3d9c6c..873d62f32e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -5,8 +5,8 @@ """ from copy import deepcopy -from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired +from dataclasses import dataclass, replace +from typing import ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict @@ -17,6 +17,7 @@ ArrayArrayCodec, DataTypeEntity, Opaque, + canonicalized, written, ) from zarr_metadata.v3._parts import ArrayParts @@ -141,6 +142,10 @@ class CastValueCodec(ArrayArrayCodec): identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME + def canonical(self) -> Self: + """The target data type in its own canonical form.""" + return replace(self, data_type=canonicalized(self.data_type)) + def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 68b279a2b7..147914235e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -4,8 +4,8 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html """ -from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired +from dataclasses import dataclass, replace +from typing import ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict @@ -17,6 +17,7 @@ ArrayBytesCodec, CodecEntity, Opaque, + canonicalized, problem, written, ) @@ -126,6 +127,14 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) + def canonical(self) -> Self: + """Each pipeline's codecs in their own canonical form.""" + return replace( + self, + codecs=tuple(canonicalized(codec) for codec in self.codecs), + index_codecs=tuple(canonicalized(codec) for codec in self.index_codecs), + ) + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """This shard against the array reaching it, and its two pipelines. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 029d7af4df..2d87b32c7b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -5,8 +5,8 @@ """ from collections.abc import Mapping -from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, cast +from dataclasses import dataclass, replace +from typing import ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import ReadOnly, TypedDict @@ -18,6 +18,7 @@ Loc, Opaque, StorageClass, + canonicalized, problem, written, ) @@ -160,6 +161,15 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) + def canonical(self) -> Self: + """Each field's data type in its own canonical form.""" + return replace( + self, + fields=tuple( + replace(field, data_type=canonicalized(field.data_type)) for field in self.fields + ), + ) + def storage_class(self) -> StorageClass | None: """The widest class among the fields. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index cc00015201..5532e0b946 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -14,13 +14,13 @@ model is not a failure: it arrives as an `Opaque` marked `out_of_scope`, for the reader to resolve elsewhere. - from zarr_metadata.v3.entity import ArrayDocumentV3, CodecEntity + from zarr_metadata.v3.entity import ArrayBytesCodec, ArrayDocumentV3, CodecEntity array = ArrayDocumentV3.from_json(json.loads(raw)) # or raises array.parts.grid.rank for codec in array.codecs: if isinstance(codec, CodecEntity): - codec.kind # 'array_bytes' + isinstance(codec, ArrayBytesCodec) # its pipeline position is its base class else: codec.json, codec.reason # 'out_of_scope': resolve it yourself @@ -94,9 +94,11 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: ones JSON takes: `int`, `float` (any JSON number), `bool`, `str`, `JSONValue`, a `Literal` of names, `tuple[T, ...]` or `tuple[T1, T2]`, a TypedDict or dataclass record, `Mapping[str, V]`, a `NewType`, and a -nested entity -- always with `Opaque`, `inner: CodecEntity | Opaque`, -because that is what the field holds when the inner name is out of -scope. Anything else is refused at class creation. A required member +nested entity, always as `inner: CodecEntity | Opaque`, because that is +what the field holds when the inner name is out of scope -- at any +depth, as an array's element or a record's field, and read in the scope +the containing entity is read in. Anything else is refused at class +creation. A required member has no default; an optional one is `| UNSET = UNSET`, so absence stays distinct from a JSON `null`, and a member that means something when absent is read that way where it is used, not defaulted. @@ -115,10 +117,11 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: abstract: the entity as a document writes it, as a literal of its own TypedDict, which pyright holds to that type -- the bare name when every member is absent, the object otherwise, a contained entity through -`written`. `coerce` and `canonical` are written once in the base; an -entity whose own members have two spellings that mean the same overrides -`simplified`, which `canonical` calls (overriding `canonical` itself is -refused). Then, by kind: +`written`. `canonical`, the entity in its simplest equivalent form: the +entity itself by default, overridden where two spellings of its members +mean the same, and in an entity that contains entities to put those in +canonical form -- `replace(self, inner=canonicalized(self.inner))`. +`coerce` is written once in the base. Then, by kind: - Every entity: `identifier`, the name it is registered under. A family -- one class for every `acme.fixedN` -- overrides `accepts(name)` and @@ -142,14 +145,16 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: `ChunkGridEntity`. What a kind leaves abstract, registration refuses an entity for not -defining; the other mistakes an author would not otherwise see -- a -`scalar_storage` outside the listed values, a nested field without -`Opaque`, a class without `@dataclass`, an entity subclassing -`MetadataEntity` or `CodecEntity` instead of a kind -- are refused at -class creation or registration with a message that says what to write. -A scope reads what a class is off the class: its kind is its base, its -key is its `identifier`, so `extended_with` takes the classes and nothing -can be misfiled. +defining, as it refuses a class without `@dataclass` and a codec +subclassing `CodecEntity` instead of a kind; class creation refuses a +field whose annotation is not a shape JSON takes -- a nested entity +without `Opaque` among them -- and a class variable a base annotates and +nothing sets. Each says what to write. Everything else an author could +get wrong, pyright says in the editor: the fields, the class variables +and the kind's abstract methods are ordinary typed Python. A scope reads +what a class is off the class: its kind is its base, its key is its +`identifier`, so `extended_with` takes the classes and nothing can be +misfiled. **Naming the JSON type.** The return annotation of `to_json` -- above, `AcmeLz4Object | Literal["acme.lz4"]` -- is the entity's own JSON type, @@ -199,7 +204,6 @@ class creation or registration with a message that says what to write. ChunkGridEntity, ChunkKeyEncodingEntity, CodecEntity, - CodecKind, Coerced, DataTypeEntity, Loc, @@ -207,6 +211,7 @@ class creation or registration with a message that says what to write. Opaque, StorageClass, StorageTransformerEntity, + canonicalized, is_integer, named_configuration, problem, @@ -236,7 +241,6 @@ class creation or registration with a message that says what to write. "ChunkGridEntity", "ChunkKeyEncodingEntity", "CodecEntity", - "CodecKind", "Coerced", "ComplexDataType", "Context", @@ -255,6 +259,7 @@ class creation or registration with a message that says what to write. "StorageTransformerEntity", "ValidationProblem", "ZarrV3MetadataFieldJSON", + "canonicalized", "chain_problems", "is_integer", "named_configuration", diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py index cd4ea4d6d8..953e55ea90 100644 --- a/packages/zarr-metadata/tests/rules/test_chain_properties.py +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -27,7 +27,12 @@ valid_documents, ) from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.v3.entity import CORE_AND_EXTENSIONS, CodecEntity +from zarr_metadata.v3.entity import ( + CORE_AND_EXTENSIONS, + ArrayArrayCodec, + ArrayBytesCodec, + CodecEntity, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -58,12 +63,12 @@ def test_the_strategies_cover_every_codec_the_package_models() -> None: if attribute.endswith("_CODEC_NAME") and isinstance(value, str) } assert modelled == drawn - for kinds, expected in ((ARRAY_ARRAY, "array_array"), (ARRAY_BYTES, "array_bytes")): + for kinds, expected in ((ARRAY_ARRAY, ArrayArrayCodec), (ARRAY_BYTES, ArrayBytesCodec)): for entry in kinds: name = entry.__annotations__["name"].__args__[0] entity = CORE_AND_EXTENSIONS.resolve(CodecEntity, name) assert entity is not None, name - assert entity.kind == expected + assert issubclass(entity, expected) @given(codec_chains()) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index cba836eb57..c78410085c 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -290,7 +290,6 @@ def test_all_is_grouped_and_unique() -> None: "Canonical", "CastOutOfRangeMode", "CastRoundingMode", - "CodecKind", "StorageClass", "Loc", "Extents", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index c9c956a0b6..07e6aca0b2 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -31,6 +31,7 @@ Opaque, ValidationProblem, ZarrV3MetadataFieldJSON, + canonicalized, problem, written, ) @@ -76,9 +77,13 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) - def simplified(self) -> Self: - """An offset of 0 is the identity, and absent says the same.""" - return replace(self, offset=UNSET) if self.offset == 0 else self + def canonical(self) -> Self: + """An offset of 0 is the identity, and absent says the same; `dtype` in its own form.""" + return replace( + self, + offset=UNSET if self.offset == 0 else self.offset, + dtype=UNSET if self.dtype is UNSET else canonicalized(self.dtype), + ) def to_json(self) -> AcmeAffineObject: configuration: AcmeAffineConfiguration = {"scale": self.scale} diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index ff77d45b5b..4b8b9b8a61 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -744,7 +744,7 @@ class AcmeShardCache(StorageTransformerEntity): identifier: ClassVar[str] = "acme.shard_cache" - def simplified(self) -> Self: + def canonical(self) -> Self: return dataclasses.replace(self, verbose=UNSET) def to_json(self) -> ZarrV3MetadataFieldJSON: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 9257c279b7..4a20de3b28 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -30,7 +30,6 @@ ChunkGridEntity, ChunkKeyEncodingEntity, CodecEntity, - CodecKind, Context, DataTypeEntity, IntegerDataType, @@ -41,6 +40,7 @@ StorageClass, ValidationProblem, ZarrV3MetadataFieldJSON, + canonicalized, problem, written, ) @@ -184,19 +184,27 @@ def test_the_entity_layer_answers_what_a_reader_needs() -> None: assert parts.grid.axis(0) == frozenset({32}) -def test_error_an_optional_member_defaults_to_unset() -> None: - # Otherwise every instance emits it, the bare-name spelling becomes - # unreachable, and a canonicalized document gains a member the writer - # never wrote. - with pytest.raises(TypeError, match="a default other than UNSET"): +def test_an_absent_optional_member_is_read_as_unset_whatever_its_default() -> None: + # A default is for hand construction; what a document left out is + # `UNSET`, so no field's default decides what a document said. + @dataclass(frozen=True) + class Defaulted(BytesBytesCodec): + level: int | UNSET = 3 - @dataclass(frozen=True) - class Inventive(BytesBytesCodec): - # Optional by its type, so the annotation and the default agree - # on that much; it is the default's value that is wrong. - level: int | UNSET = 3 + identifier: ClassVar[str] = "acme.defaulted" + + def to_json(self) -> ZarrV3MetadataFieldJSON: + if self.level is UNSET: + return "acme.defaulted" + return {"name": "acme.defaulted", "configuration": {"level": self.level}} - identifier: ClassVar[str] = "acme.inventive" + assert Defaulted().level == 3 + codec, problems = CORE_AND_EXTENSIONS.extended_with(Defaulted).coerce( + CodecEntity, "acme.defaulted" + ) + assert problems == () + assert isinstance(codec, Defaulted) + assert codec.level is UNSET def test_a_reader_gets_entities_or_an_exception() -> None: @@ -260,19 +268,6 @@ def test_a_reader_can_choose_its_own_scope() -> None: assert in_scope.acceleration == 4 -def test_error_a_field_may_not_shadow_a_class_variable() -> None: - # A field of that name goes into the configuration and into the JSON, - # while the class variable it shadows is what the rest of the layer - # reads -- so the entity would claim one thing and behave as another. - with pytest.raises(TypeError, match="shadowing a class variable"): - - @dataclass(frozen=True) - class Negotiable(BytesBytesCodec): - kind: str = "bytes_bytes" # pyright: ignore[reportIncompatibleVariableOverride] - - identifier: ClassVar[str] = "acme.negotiable" - - def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: # `bounds` is annotated on `IntegerDataType` and bound by none of it, # so every concrete integer type owes one. Nothing lists it: the @@ -343,8 +338,7 @@ class Structured(BytesBytesCodec): identifier: ClassVar[str] = "acme.structured" -# A third-party codec that contains another codec: the case that used to -# need `prepare`, `configuration` and `canonical` written by hand. +# A third-party codec that contains another codec. @dataclass(frozen=True) class AcmeWrapperCodec(BytesBytesCodec): """A codec that applies another codec after its own step.""" @@ -353,14 +347,18 @@ class AcmeWrapperCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.wrapper" + def canonical(self) -> Self: + return replace(self, inner=canonicalized(self.inner)) + def to_json(self) -> ZarrV3MetadataFieldJSON: return {"name": "acme.wrapper", "configuration": {"inner": written(self.inner)}} -def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> None: - # `inner: CodecEntity | Opaque` is the whole declaration. Reading it - # in scope, writing it back, and canonicalizing through it all follow - # from the annotation, so a wrapper is as short to write as a leaf. +def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: + # `inner: CodecEntity | Opaque` is the whole declaration of the + # reading: the inner codec is resolved in the scope the wrapper is + # read in, and its problems are located inside. Writing and + # canonicalizing it are the wrapper's own two lines. scope = CORE_AND_EXTENSIONS.extended_with(AcmeWrapperCodec) entry = { "name": "acme.wrapper", @@ -414,23 +412,9 @@ def test_a_third_party_entity_containing_entities_writes_nothing_for_it() -> Non assert inner.typesize is UNSET -def test_error_an_entity_may_not_override_canonical() -> None: - # `canonical` is the walk into contained entities, read off the - # annotations; an override could lose it. The entity's own rewrite - # goes in `simplified`. - with pytest.raises(TypeError, match="put the entity's own rewrite in `simplified`"): - - @dataclass(frozen=True) - class Rewriter(BytesBytesCodec): - identifier: ClassVar[str] = "acme.rewriter" - - def canonical(self) -> Self: # pyright: ignore[reportIncompatibleMethodOverride] - return self - - -def test_simplified_composes_with_the_walk_into_contained_entities() -> None: - # An entity that contains an entity and rewrites its own members gets - # both from `canonical` -- the contained blosc loses the `typesize` +def test_canonical_is_the_entity_s_own_and_reaches_what_it_contains() -> None: + # An entity that contains an entity and rewrites its own members does + # both in one `canonical` -- the contained blosc loses the `typesize` # that `noshuffle` ignores, and the frame of 0 that means "unframed" # is dropped -- with nothing to call `super()` for. @dataclass(frozen=True) @@ -440,8 +424,12 @@ class AcmeFramedCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.framed" - def simplified(self) -> Self: - return self if self.frame != 0 else replace(self, frame=UNSET) + def canonical(self) -> Self: + return replace( + self, + inner=canonicalized(self.inner), + frame=UNSET if self.frame == 0 else self.frame, + ) def to_json(self) -> ZarrV3MetadataFieldJSON: configuration: dict[str, JSONValue] = {"inner": written(self.inner)} @@ -455,10 +443,10 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: assert framed.inner is blosc # a transformation, not a mutation -def test_error_a_nested_field_needs_an_entity_kind_with_a_point() -> None: - # `MetadataEntity` is registered at no single point, so a field typed - # as one could not be resolved through any scope. - with pytest.raises(TypeError, match="is of no kind; annotate it with a codec kind"): +def test_error_a_nested_field_names_a_kind() -> None: + # `MetadataEntity` is of no kind, so a field typed as one could not + # be resolved through any scope. + with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): @dataclass(frozen=True) class Vague(BytesBytesCodec): @@ -531,10 +519,9 @@ def test_a_rule_about_a_member_is_post_init() -> None: assert [p.kind for p in problems] == ["invalid_type"] -def test_a_slotted_entity_is_compiled_once() -> None: - # `@dataclass(slots=True)` builds the class twice; the second pass - # arrives with the derived tables already on it and must not be - # refused as having declared them. +def test_a_slotted_entity_is_accepted() -> None: + # `@dataclass(slots=True)` builds the class twice, so class creation + # sees it twice; the second time its members are slot descriptors. @dataclass(frozen=True, slots=True) class AcmeSlotted(BytesBytesCodec): level: int @@ -611,7 +598,7 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: def test_error_a_nested_field_admits_opaque() -> None: # What the field holds when the inner name is out of scope. - with pytest.raises(TypeError, match="inner holds an entity but does not admit Opaque"): + with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): @dataclass(frozen=True) class Closed(BytesBytesCodec): @@ -637,14 +624,19 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: def test_error_a_codec_is_of_a_kind() -> None: + # The kind classes say what a codec does to the array; registration + # refuses one that skipped them. + @dataclass(frozen=True) + class Kindless(CodecEntity): + identifier: ClassVar[str] = "acme.kindless" + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.kindless" + with pytest.raises( TypeError, match="subclasses CodecEntity directly; subclass ArrayArrayCodec" ): - - @dataclass(frozen=True) - class Kindless(CodecEntity): - identifier: ClassVar[str] = "acme.kindless" - kind: ClassVar[CodecKind] = "bytes_bytes" + CORE_AND_EXTENSIONS.extended_with(Kindless) def test_error_a_data_type_judges_its_fill_values() -> None: @@ -661,19 +653,6 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: CORE_AND_EXTENSIONS.extended_with(Lax) -def test_error_a_literal_class_variable_holds_a_listed_value() -> None: - # `bytes` asks a data type's storage class and has nothing to say - # about a fourth value: the endian rule would silently not apply. - with pytest.raises( - TypeError, match="sets scalar_storage = 'sixteen_bytes', which is not one of" - ): - - @dataclass(frozen=True) - class Wide(DataTypeEntity): - identifier: ClassVar[str] = "acme.wide" - scalar_storage: ClassVar[StorageClass] = "sixteen_bytes" # pyright: ignore[reportAssignmentType] - - def test_error_a_list_of_problem_tuples_is_refused() -> None: # `problem()` returns a one-element tuple; a list of those would pass # the constructor and fail inside `coerce`, far from the mistake. From c8b71321f537334744fcf88281601c4e9d18b2da Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 16:55:33 +0200 Subject: [PATCH 085/107] refactor(zarr-metadata): an entity's rules are a function of the instance that yields problems Every check that lived in an entity's `__post_init__` is a module-level function of the instance -- `blosc_problems(codec)` -- that yields each problem as it finds it, bound on the class as `problems`. One function, two consumers: the constructor takes the first problem it yields and raises `MetadataValidationError`, so `BloscCodec(clevel=99)` still refuses; `coerce` runs it to the end and reports every problem in the document. Anyone holding an entity may run it too, and stop or collect as they need. `coerce` builds the instance without asking the constructor -- the members are already parsed and typed -- and asks `problems` itself, so a document's every problem is reported where the constructor would stop. Class creation refuses an entity that defines `__post_init__`, because nothing would run it on a document. The two numpy time types declare their shared configuration and rule on the family. No verdict or problem changes over the 40,000-document corpus. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.10.md | 11 +- .../zarr-metadata/changes/4379.feature.7.md | 14 ++- .../zarr-metadata/changes/4379.feature.md | 5 +- packages/zarr-metadata/changes/4379.misc.2.md | 2 +- .../src/zarr_metadata/v3/_entity.py | 87 +++++++++++---- .../v3/chunk_grid/rectilinear.py | 44 ++++---- .../zarr_metadata/v3/chunk_grid/regular.py | 26 ++--- .../src/zarr_metadata/v3/codec/blosc.py | 85 +++++++------- .../src/zarr_metadata/v3/codec/gzip.py | 23 ++-- .../zarr_metadata/v3/codec/scale_offset.py | 38 ++++--- .../v3/codec/sharding_indexed.py | 29 +++-- .../src/zarr_metadata/v3/codec/transpose.py | 36 +++--- .../src/zarr_metadata/v3/codec/zstd.py | 27 +++-- .../zarr_metadata/v3/data_type/_families.py | 22 +++- .../v3/data_type/numpy_datetime64.py | 17 --- .../v3/data_type/numpy_timedelta64.py | 16 --- .../src/zarr_metadata/v3/data_type/raw.py | 25 ++--- .../src/zarr_metadata/v3/data_type/struct.py | 88 +++++++-------- .../src/zarr_metadata/v3/entity.py | 49 +++++---- .../tests/v3/test_acme_affine.py | 35 +++--- .../tests/v3/test_acme_decimal.py | 56 +++++----- .../tests/v3/test_extension_api.py | 104 ++++++++++++++---- 22 files changed, 451 insertions(+), 388 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index bc540048e7..b65416a179 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -5,11 +5,12 @@ such thing as an invalid entity to hold. Code that wants unvalidated metadata already has somewhere to put it: `Opaque`, which carries the JSON verbatim. -One piece: the entity's `__post_init__`, in plain code, which collects -every value the spec disallows and raises `MetadataValidationError` once. -The constructor asks, then builds; `coerce` builds the same way, catches -the same error, and reports its problems in the document instead of -raising them. +One piece: a function of the instance that yields the values the spec +disallows as it finds them, bound on the class as `problems`. The +constructor stops at the first and raises `MetadataValidationError`; +`coerce` runs it to the end and reports every problem in the document +instead of raising; a consumer holding an entity may run it too, and +stop or collect as it likes. `must_understand` moves to a class variable and out of the configuration entirely. It is a property of the *kind* of metadata -- a codec is diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 6a165ab0a1..b47bef257b 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -69,17 +69,19 @@ refused at class creation: no scope could place the one, and the other lies about what the field holds when the inner name is out of scope. Everything finer than a type -- a bound, a rule about one member, a -rule that reads two members together -- is the entity's own -`__post_init__`, in plain code: it collects every problem it finds, -located relative to the configuration, and raises `MetadataValidationError` -once, so `BloscCodec(clevel=99)` raises; `coerce` catches the same -error and reports the problems in the document instead. The space of +rule that reads two members together -- is a function of the entity's +instance, in plain code, that yields each problem as it finds it, +located relative to the configuration, and is bound on the class as +`problems`. The constructor stops at the first, so `BloscCodec(clevel=99)` +raises `MetadataValidationError`; `coerce` runs it to the end and +reports every problem in the document; a consumer holding an entity may +run it too, stopping or collecting as it needs. The space of refinements is too wide to capture statically, and reading them in one place per entity is what will show where a shared form is worth having. The set of annotation shapes the parser reads is closed: the shapes JSON takes, and no others. A field annotation outside them is refused at class creation, and the field is written as one of them instead, with -any finer rule in `__post_init__`. The parser is one module that knows +any finer rule in the function bound as `problems`. The parser is one module that knows nothing of entities; the entity layer hands it the one shape of its own, a field holding an entity, as a leaf it reads at any depth. diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index 98aa42cf9c..1da2ed4dbb 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -9,8 +9,9 @@ data type, chunk grid and chunk key encoding — is a class that answers for itself: - `coerce` reads raw metadata into the entity, or says why it is not one; -- `__post_init__` refuses the values the spec disallows, collecting every - problem and raising once; `coerce` reports the same problems instead; +- `problems`, a function of the instance bound on the class, yields the + values the spec disallows as it finds them; the constructor stops at + the first and raises, and `coerce` reports every one instead; - `to_json` writes it back as it was read, and `canonical` gives its simplest equivalent spelling; - `incoming_problems`, `shape_problems`, `fill_value_problems` and diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index bebc637976..5100ffb04f 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -43,7 +43,7 @@ them when it reads a document. Four adversarial reviews -- two extension authors writing a codec and a data type against the door alone, a design review, an onboarding review -- drove the last round. `coerce` no longer builds an entity around a -member that could not be read, which had `__post_init__` judging the +member that could not be read, which had the entity's rules judging the hole; nested entities are resolved whatever else was found, so their problems come out in the same pass; `FROM_NAME` fields are filled by the base; `to_json` on a document writes only the fields it had; `float` diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 969987f5e7..362e208959 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -6,9 +6,10 @@ configuration against them, member by member, with `_typed_json`. A field typed `CodecEntity | Opaque` holds another entity, read through the scope the containing one is read in. Everything finer than a type -- -a bound, a rule about a member, members read together -- is the entity's -own `__post_init__`, which collects every problem it finds and raises -once; `coerce` reports those instead of raising. +a bound, a rule about a member, members read together -- is a function +of the entity's instance that yields problems as it finds them, bound +on the class as `problems`; the constructor stops at the first, +`coerce` reports every one. What an entity writes and what it simplifies to are its own too: `to_json` is abstract, a literal of the entity's JSON type, and @@ -49,7 +50,7 @@ ) if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterator, Sequence from typing import Self from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON @@ -281,13 +282,13 @@ class MetadataEntity(ABC): mapping instead: `MappingProxyType` is unhashable too, and anything else stops `json.dumps` from serializing what `to_json` returns. - A subclass writes its fields; a `__post_init__` where the spec has - something to say beyond their types, collecting every problem and - raising `MetadataValidationError` once -- so `BloscCodec(clevel=99)` - raises, and `coerce` reports the same problems instead; `to_json`, a - literal of its JSON type; and `canonical` where two spellings of its - members mean the same. `coerce` is written once here, against what - the fields say. + A subclass writes its fields; where the spec has something to say + beyond their types, a function of the instance that yields problems, + bound as `problems` -- so `BloscCodec(clevel=99)` raises on the + first, and `coerce` reports every one instead; `to_json`, a literal + of its JSON type; and `canonical` where two spellings of its members + mean the same. `coerce` is written once here, against what the + fields say. """ identifier: ClassVar[str] @@ -301,13 +302,14 @@ class MetadataEntity(ABC): def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: """Refuse, at class creation, an entity this layer could not read. - Two things type-check cleanly and then go wrong somewhere that + Three things type-check cleanly and then go wrong somewhere that will not name the class: a field whose annotation is not a shape - JSON takes, which `coerce` could not parse, and a class variable - a base annotates and nothing sets -- `identifier` for every - entity, `bounds` for an integer type -- which the first lookup - would fail. An import-time error in the extension's own module is - the one place the author is looking. + JSON takes, which `coerce` could not parse; a `__post_init__` of + the entity's own, whose rules `coerce` would never ask; and a + class variable a base annotates and nothing sets -- `identifier` + for every entity, `bounds` for an integer type -- which the first + lookup would fail. An import-time error in the extension's own + module is the one place the author is looking. `base=True` for a class that exists to add a class variable rather than to be an entity -- `CodecEntity`, `IntegerDataType`. @@ -333,7 +335,14 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " "record, Mapping[str, V], a NewType, or an entity kind with Opaque " "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in `__post_init__`" + "rule in the function bound as `problems`" + ) + raise TypeError(msg) + if "__post_init__" in vars(cls): + msg = ( + f"{cls.__name__} defines __post_init__; write its rules as a function of the " + "instance that yields problems and bind it as `problems = `: the " + "constructor stops at the first problem it yields, `coerce` reports every one" ) raise TypeError(msg) annotated = declared_class_vars(cls) @@ -348,6 +357,36 @@ def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: ) raise TypeError(msg) + def problems(self, /) -> Iterator[ValidationProblem]: + """Every reason this entity's values are not allowed, yielded as found. + + The entity's own rules -- a bound, a rule about one member, + members read together -- written as a function of the instance + and bound on the class: `problems = blosc_problems`. Locations + are relative to the configuration. A consumer stops at the first + or collects them all, as it needs: the constructor stops at the + first, `coerce` collects every one. Default: none. + """ + yield from () + + def __post_init__(self) -> None: + """Refuse the first problem `problems` finds, so `BloscCodec(clevel=99)` raises.""" + first = next(self.problems(), None) + if first is not None: + raise MetadataValidationError((first,)) + + @classmethod + def _unchecked(cls, members: Mapping[str, object]) -> Self: + """The instance `cls(**members)` would build, without asking `problems`. + + For `coerce`, which asks `problems` itself and reports every one, + where the constructor stops at the first. + """ + entity = object.__new__(cls) + for name, value in members.items(): + object.__setattr__(entity, name, value) + return entity + @classmethod def accepts(cls, name: str) -> bool: """Whether `name` denotes this entity. @@ -418,12 +457,12 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # An unknown key is survivable; a member that could not be # read is a hole, and judging around it would be guessing. return None, found - try: - entity = cls(**members) - except MetadataValidationError as refused: - # `__post_init__` found values the spec disallows: reported - # rather than raised, located under the configuration. - return None, (*found, *within((), refused.problems)) + entity = cls._unchecked(members) + refused = within((), tuple(entity.problems())) + if len(refused) != 0: + # Values the spec disallows: reported rather than raised, + # every one, located under the configuration. + return None, (*found, *refused) if any(entry.kind != "unknown_key" for entry in nested): # A contained entity could not be read. This entity's own # rules ran -- an invalid inner is an `Opaque`, as an diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 024832a498..74457916bc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -9,7 +9,7 @@ from typing_extensions import TypedDict -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, Loc, @@ -19,7 +19,7 @@ from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterator, Sequence RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" @@ -69,8 +69,8 @@ class RectilinearChunkGridObject(TypedDict, closed=True): """ -def _not_positive(loc: Loc, value: int) -> tuple[ValidationProblem, ...]: - return problem(loc, f"expected an integer >= 1, got {value}", "invalid_value") +def _not_positive(loc: Loc, value: int) -> ValidationProblem: + return ValidationProblem(loc, f"expected an integer >= 1, got {value}", "invalid_value") def canonical_dim_spec(spec: RectilinearDimSpec) -> RectilinearDimSpec: @@ -166,6 +166,23 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: return frozenset(lengths) if len(lengths) != 0 else None +def rectilinear_problems(grid: "RectilinearChunkGrid", /) -> "Iterator[ValidationProblem]": + """Every extent, and every run's length and count, is at least 1.""" + for axis, spec in enumerate(grid.chunk_shapes): + if isinstance(spec, int): + if spec < 1: + yield _not_positive(("chunk_shapes", axis), spec) + continue + for index, entry in enumerate(spec): + if isinstance(entry, int): + if entry < 1: + yield _not_positive(("chunk_shapes", axis, index), entry) + continue + for position, value in enumerate(entry): + if value < 1: + yield _not_positive(("chunk_shapes", axis, index, position), value) + + @dataclass(frozen=True) class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" @@ -175,24 +192,7 @@ class RectilinearChunkGrid(ChunkGridEntity): identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME - def __post_init__(self) -> None: - """Every extent, and every run's length and count, is at least 1.""" - found: list[ValidationProblem] = [] - for axis, spec in enumerate(self.chunk_shapes): - if isinstance(spec, int): - if spec < 1: - found.extend(_not_positive(("chunk_shapes", axis), spec)) - continue - for index, entry in enumerate(spec): - if isinstance(entry, int): - if entry < 1: - found.extend(_not_positive(("chunk_shapes", axis, index), entry)) - continue - for position, value in enumerate(entry): - if value < 1: - found.extend(_not_positive(("chunk_shapes", axis, index, position), value)) - if len(found) != 0: - raise MetadataValidationError(found) + problems = rectilinear_problems def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index fc2e608f64..a59f376bb5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -9,7 +9,7 @@ from typing_extensions import TypedDict -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, problem, @@ -17,7 +17,7 @@ from zarr_metadata.v3._parts import ChunkGrid if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Iterator, Sequence REGULAR_CHUNK_GRID_NAME: Final = "regular" @@ -60,6 +60,14 @@ class RegularChunkGridObject(TypedDict, closed=True): ] +def regular_problems(grid: "RegularChunkGrid", /) -> "Iterator[ValidationProblem]": + for index, extent in enumerate(grid.chunk_shape): + if extent < 1: + yield ValidationProblem( + ("chunk_shape", index), f"expected an integer >= 1, got {extent}", "invalid_value" + ) + + @dataclass(frozen=True) class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" @@ -68,19 +76,7 @@ class RegularChunkGrid(ChunkGridEntity): identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME - def __post_init__(self) -> None: - found: list[ValidationProblem] = [] - for index, extent in enumerate(self.chunk_shape): - if extent < 1: - found.extend( - problem( - ("chunk_shape", index), - f"expected an integer >= 1, got {extent}", - "invalid_value", - ) - ) - if len(found) != 0: - raise MetadataValidationError(found) + problems = regular_problems def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 049c4a89d1..90f26d3297 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -5,17 +5,19 @@ """ from dataclasses import dataclass, replace -from typing import ClassVar, Final, Literal, NotRequired, Self +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( BytesBytesCodec, - problem, ) +if TYPE_CHECKING: + from collections.abc import Iterator + BLOSC_CODEC_NAME: Final = "blosc" """The `name` field value of the `blosc` codec.""" @@ -87,6 +89,36 @@ class BloscCodecObject(TypedDict, closed=True): ] +def blosc_problems(codec: "BloscCodec", /) -> "Iterator[ValidationProblem]": + """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. + + Under `noshuffle` the spec says of `typesize` that "the value is + ignored", and `canonical` drops it; under either shuffle it is + required, and positive. + """ + if not 0 <= codec.clevel <= 9: + yield ValidationProblem( + ("clevel",), f"expected an integer in [0, 9], got {codec.clevel}", "invalid_value" + ) + if codec.blocksize < 0: + yield ValidationProblem( + ("blocksize",), f"expected an integer >= 0, got {codec.blocksize}", "invalid_value" + ) + if codec.shuffle != BLOSC_NO_SHUFFLE: + if codec.typesize is UNSET: + yield ValidationProblem( + ("typesize",), + f"typesize is required when shuffle is {codec.shuffle!r}", + "missing_key", + ) + elif codec.typesize < 1: + yield ValidationProblem( + ("typesize",), + f"expected a positive integer, got {codec.typesize}", + "invalid_value", + ) + + @dataclass(frozen=True) class BloscCodec(BytesBytesCodec): """The `blosc` codec, coerced from its metadata. @@ -106,51 +138,8 @@ class BloscCodec(BytesBytesCodec): variable_size: ClassVar[bool] = True # Every member is required but `typesize`, which only means something - # when shuffling; `problems` is where that conditional lives. - - def __post_init__(self) -> None: - """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. - - Under `noshuffle` the spec says of `typesize` that "the value is - ignored", and `canonical` drops it; under either shuffle it is - required, and positive. - """ - found: list[ValidationProblem] = [] - if not 0 <= self.clevel <= 9: - found.extend( - problem( - ("clevel",), - f"expected an integer in [0, 9], got {self.clevel}", - "invalid_value", - ) - ) - if self.blocksize < 0: - found.extend( - problem( - ("blocksize",), - f"expected an integer >= 0, got {self.blocksize}", - "invalid_value", - ) - ) - if self.shuffle != BLOSC_NO_SHUFFLE: - if self.typesize is UNSET: - found.extend( - problem( - ("typesize",), - f"typesize is required when shuffle is {self.shuffle!r}", - "missing_key", - ) - ) - elif self.typesize < 1: - found.extend( - problem( - ("typesize",), - f"expected a positive integer, got {self.typesize}", - "invalid_value", - ) - ) - if len(found) != 0: - raise MetadataValidationError(found) + # when shuffling; `blosc_problems` is where that conditional lives. + problems = blosc_problems def canonical(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 6ff873e5ff..3acc707e9d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -5,16 +5,18 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict -from zarr_metadata.model._validation import MetadataValidationError +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( BytesBytesCodec, - problem, ) +if TYPE_CHECKING: + from collections.abc import Iterator + GZIP_CODEC_NAME: Final = "gzip" """The `name` field value of the `gzip` codec.""" @@ -65,6 +67,13 @@ class GzipCodecObject(TypedDict, closed=True): ] +def gzip_problems(codec: "GzipCodec", /) -> "Iterator[ValidationProblem]": + if not 0 <= codec.level <= 9: + yield ValidationProblem( + ("level",), f"expected an integer in [0, 9], got {codec.level}", "invalid_value" + ) + + @dataclass(frozen=True) class GzipCodec(BytesBytesCodec): """The `gzip` codec, coerced from its metadata.""" @@ -74,13 +83,7 @@ class GzipCodec(BytesBytesCodec): identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True - def __post_init__(self) -> None: - if not 0 <= self.level <= 9: - raise MetadataValidationError( - problem( - ("level",), f"expected an integer in [0, 9], got {self.level}", "invalid_value" - ) - ) + problems = gzip_problems def to_json(self) -> GzipCodecObject: return {"name": "gzip", "configuration": {"level": self.level}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index bfe4ce8668..00126f4c4a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -6,19 +6,21 @@ from copy import deepcopy from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ArrayArrayCodec, - problem, ) from zarr_metadata.v3._parts import ArrayParts +if TYPE_CHECKING: + from collections.abc import Iterator + SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" """The `name` field value of the `scale_offset` codec.""" @@ -73,6 +75,20 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): ] +def scale_offset_problems(codec: "ScaleOffsetCodec", /) -> "Iterator[ValidationProblem]": + """Each value is a scalar of the array's type, so neither is null. + + The registry says each is "JSON-encoded per the input array's + fill-value rules", and no data type admits `null` as a fill value. + Which scalar it should be needs the data type, so that part is the + document's question, not this codec's. + """ + if codec.offset is None: + yield ValidationProblem(("offset",), "expected a scalar, got null", "invalid_value") + if codec.scale is None: + yield ValidationProblem(("scale",), "expected a scalar, got null", "invalid_value") + + @dataclass(frozen=True) class ScaleOffsetCodec(ArrayArrayCodec): """The `scale_offset` codec, coerced from its metadata. @@ -87,21 +103,7 @@ class ScaleOffsetCodec(ArrayArrayCodec): identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME - def __post_init__(self) -> None: - """Each value is a scalar of the array's type, so neither is null. - - The registry says each is "JSON-encoded per the input array's - fill-value rules", and no data type admits `null` as a fill value. - Which scalar it should be needs the data type, so that part is the - document's question, not this codec's. - """ - found: list[ValidationProblem] = [] - if self.offset is None: - found.extend(problem(("offset",), "expected a scalar, got null", "invalid_value")) - if self.scale is None: - found.extend(problem(("scale",), "expected a scalar, got null", "invalid_value")) - if len(found) != 0: - raise MetadataValidationError(found) + problems = scale_offset_problems def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same array, element for element. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 147914235e..0750b27f2f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -5,12 +5,12 @@ """ from dataclasses import dataclass, replace -from typing import ClassVar, Final, Literal, NotRequired, Self +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( @@ -29,6 +29,9 @@ ) from zarr_metadata.v3.data_type.uint64 import Uint64DataType +if TYPE_CHECKING: + from collections.abc import Iterator + SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -96,6 +99,14 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): ] +def sharding_problems(codec: "ShardingIndexedCodec", /) -> "Iterator[ValidationProblem]": + for index, extent in enumerate(codec.chunk_shape): + if extent < 1: + yield ValidationProblem( + ("chunk_shape", index), f"expected an integer >= 1, got {extent}", "invalid_value" + ) + + @dataclass(frozen=True) class ShardingIndexedCodec(ArrayBytesCodec): """The `sharding_indexed` codec, coerced from its metadata. @@ -113,19 +124,7 @@ class ShardingIndexedCodec(ArrayBytesCodec): identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME variable_size: ClassVar[bool] = True - def __post_init__(self) -> None: - found: list[ValidationProblem] = [] - for index, extent in enumerate(self.chunk_shape): - if extent < 1: - found.extend( - problem( - ("chunk_shape", index), - f"expected an integer >= 1, got {extent}", - "invalid_value", - ) - ) - if len(found) != 0: - raise MetadataValidationError(found) + problems = sharding_problems def canonical(self) -> Self: """Each pipeline's codecs in their own canonical form.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 5a379fdf5e..4929aa0cd5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -5,17 +5,20 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ArrayArrayCodec, problem, ) from zarr_metadata.v3._parts import ArrayParts +if TYPE_CHECKING: + from collections.abc import Iterator + TRANSPOSE_CODEC_NAME: Final = "transpose" """The `name` field value of the `transpose` codec.""" @@ -61,6 +64,20 @@ class TransposeCodecObject(TypedDict, closed=True): ] +def transpose_problems(codec: "TransposeCodec", /) -> "Iterator[ValidationProblem]": + """`order` must permute its own axes. + + Whether it permutes the *array's* axes is a different question -- it + needs the array's rank -- and the rules layer asks that one. + """ + if sorted(codec.order) != list(range(len(codec.order))): + yield ValidationProblem( + ("order",), + f"expected a permutation of 0..{len(codec.order) - 1}, got {codec.order!r}", + "invalid_value", + ) + + @dataclass(frozen=True) class TransposeCodec(ArrayArrayCodec): """The `transpose` codec, coerced from its metadata.""" @@ -69,20 +86,7 @@ class TransposeCodec(ArrayArrayCodec): identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME - def __post_init__(self) -> None: - """`order` must permute its own axes. - - Whether it permutes the *array's* axes is a different question -- - it needs the array's rank -- and the rules layer asks that one. - """ - if sorted(self.order) != list(range(len(self.order))): - raise MetadataValidationError( - problem( - ("order",), - f"expected a permutation of 0..{len(self.order) - 1}, got {self.order!r}", - "invalid_value", - ) - ) + problems = transpose_problems def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 50ff6ca2f2..2b04176a30 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -7,17 +7,19 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import MetadataValidationError +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( BytesBytesCodec, - problem, ) +if TYPE_CHECKING: + from collections.abc import Iterator + ZSTD_CODEC_NAME: Final = "zstd" """The `name` field value of the `zstd` codec.""" @@ -73,6 +75,15 @@ class ZstdCodecObject(TypedDict, closed=True): ] +def zstd_problems(codec: "ZstdCodec", /) -> "Iterator[ValidationProblem]": + if not ZSTD_MIN_LEVEL <= codec.level <= ZSTD_MAX_LEVEL: + yield ValidationProblem( + ("level",), + f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {codec.level}", + "invalid_value", + ) + + @dataclass(frozen=True) class ZstdCodec(BytesBytesCodec): """The `zstd` codec, coerced from its metadata.""" @@ -83,15 +94,7 @@ class ZstdCodec(BytesBytesCodec): identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True - def __post_init__(self) -> None: - if not ZSTD_MIN_LEVEL <= self.level <= ZSTD_MAX_LEVEL: - raise MetadataValidationError( - problem( - ("level",), - f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {self.level}", - "invalid_value", - ) - ) + problems = zstd_problems def to_json(self) -> ZstdCodecObject: configuration: ZstdCodecConfiguration = {"level": self.level} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 34cc94e51a..b37e39529f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -26,7 +26,7 @@ ) if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterator from zarr_metadata.v3._entity import Loc @@ -167,16 +167,30 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP """The largest `scale_factor` numpy stores: the field is a signed int32.""" +def numpy_time_problems(data_type: NumpyTimeDataType, /) -> Iterator[ValidationProblem]: + if not 1 <= data_type.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + yield ValidationProblem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " + f"got {data_type.scale_factor}", + "invalid_value", + ) + + @dataclass(frozen=True) class NumpyTimeDataType(DataTypeEntity, base=True): """A numpy time scalar: a signed 64-bit count of units, or `NaT`. - The vocabulary the two time types share -- the unit codes and the - scale-factor bound -- lives here with the family, so neither sibling - imports it from the other. + The two time types share their configuration -- a unit and a scale + factor -- and the rule on it, so both live here with the family and + neither sibling imports them from the other. """ + unit: NumpyTimeUnit + scale_factor: int + scalar_storage: ClassVar[StorageClass] = "multi_byte" + problems = numpy_time_problems def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: if value == "NaT": diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 3b02366051..55280e4f59 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -9,13 +9,10 @@ from typing_extensions import ReadOnly, TypedDict -from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( StorageClass, - problem, ) from zarr_metadata.v3.data_type._families import ( - NUMPY_TIME_MAX_SCALE_FACTOR, NumpyTimeDataType, NumpyTimeUnit, ) @@ -73,23 +70,9 @@ class NumpyDatetime64(TypedDict, closed=True): class NumpyDatetime64DataType(NumpyTimeDataType): """The `numpy.datetime64` data type, coerced from its metadata.""" - unit: NumpyTimeUnit - scale_factor: int - scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME - def __post_init__(self) -> None: - if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - raise MetadataValidationError( - problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " - f"got {self.scale_factor}", - "invalid_value", - ) - ) - def to_json(self) -> NumpyDatetime64: return { "name": "numpy.datetime64", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index a41bfc433c..2eb50ffba2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -9,10 +9,8 @@ from typing_extensions import ReadOnly, TypedDict -from zarr_metadata.model._validation import MetadataValidationError from zarr_metadata.v3._entity import ( StorageClass, - problem, ) from zarr_metadata.v3.data_type._families import ( NUMPY_TIME_MAX_SCALE_FACTOR, @@ -76,23 +74,9 @@ class NumpyTimedelta64(TypedDict, closed=True): class NumpyTimedelta64DataType(NumpyTimeDataType): """The `numpy.timedelta64` data type, coerced from its metadata.""" - unit: NumpyTimeUnit - scale_factor: int - scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME - def __post_init__(self) -> None: - if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - raise MetadataValidationError( - problem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " - f"got {self.scale_factor}", - "invalid_value", - ) - ) - def to_json(self) -> NumpyTimedelta64: return { "name": "numpy.timedelta64", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index e42d3e9b13..ee6b2e671f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -10,18 +10,20 @@ import re from dataclasses import dataclass -from typing import Annotated, ClassVar, Final, NewType +from typing import TYPE_CHECKING, Annotated, ClassVar, Final, NewType -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( FROM_NAME, DataTypeEntity, Loc, StorageClass, - problem, ) from zarr_metadata.v3.data_type._families import byte_values +if TYPE_CHECKING: + from collections.abc import Iterator + RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) """A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). @@ -82,17 +84,16 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: ] -def _name_problems(name: str) -> tuple[ValidationProblem, ...]: - """Why `name` is not a well-formed `r`, if it is not. +def raw_bytes_problems(data_type: "RawBytesDataType", /) -> "Iterator[ValidationProblem]": + """This family's validity is in its name, not in a configuration. "raw bits, variable size given by *, limited to be a multiple of 8" -- and zero bits is not a data type. """ try: - raw_bytes_dtype_name(name) + raw_bytes_dtype_name(data_type.data_type_name) except ValueError as error: - return problem((), str(error), "invalid_value") - return () + yield ValidationProblem((), str(error), "invalid_value") @dataclass(frozen=True) @@ -125,11 +126,7 @@ def accepts(cls, name: str) -> bool: """ return RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None - def __post_init__(self) -> None: - """This family's validity is in its name, not in a configuration.""" - found = _name_problems(self.data_type_name) - if len(found) != 0: - raise MetadataValidationError(found) + problems = raw_bytes_problems def to_json(self) -> RawBytesDataTypeName: return RawBytesDataTypeName(self.data_type_name) @@ -138,7 +135,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP """One byte value per byte of the scalar. A malformed name says nothing about how wide the scalar is, so - there is no length to check against; `problems` reports the name. + there is no length to check against; `raw_bytes_problems` reports the name. """ try: raw_bytes_dtype_name(self.data_type_name) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 2d87b32c7b..1ffc65c337 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -4,14 +4,14 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from dataclasses import dataclass, replace -from typing import ClassVar, Final, Literal, NotRequired, Self, cast +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( DataTypeEntity, @@ -23,6 +23,9 @@ written, ) +if TYPE_CHECKING: + from collections.abc import Iterator + STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -101,6 +104,39 @@ def _written_field(field: StructFieldComponent) -> StructField: return {"name": field.name, "data_type": written(field.data_type)} +def struct_problems(data_type: "StructDataType", /) -> "Iterator[ValidationProblem]": + """Names exist, are non-empty and distinct; types are fixed-size. + + A fill value addresses fields by name, and a record's layout is not + determined by a variable-length field. Nothing about a field type's + own values: it is an entity, so it exists only if those are allowed. + """ + if len(data_type.fields) == 0: + yield ValidationProblem(("fields",), "expected at least one struct field", "invalid_value") + seen: dict[str, int] = {} + for index, field in enumerate(data_type.fields): + if field.name == "": + yield ValidationProblem( + ("fields", index, "name"), "expected a non-empty field name", "invalid_value" + ) + first = seen.setdefault(field.name, index) + if first != index: + yield ValidationProblem( + ("fields", index, "name"), + f"duplicate field name {field.name!r}, already used by field {first}", + "invalid_value", + ) + if ( + isinstance(field.data_type, DataTypeEntity) + and field.data_type.storage_class() == "variable_length" + ): + yield ValidationProblem( + ("fields", index, "data_type"), + "struct fields must use fixed-size data types", + "invalid_value", + ) + + @dataclass(frozen=True) class StructDataType(DataTypeEntity): """The `struct` data type, coerced from its metadata. @@ -115,51 +151,7 @@ class StructDataType(DataTypeEntity): identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" - def __post_init__(self) -> None: - """Names exist, are non-empty and distinct; types are fixed-size. - - A fill value addresses fields by name, and a record's layout is - not determined by a variable-length field. Nothing about a field - type's own values: it is an entity, so it exists only if those - are allowed. - """ - found: list[ValidationProblem] = [] - if len(self.fields) == 0: - found.extend( - problem(("fields",), "expected at least one struct field", "invalid_value") - ) - seen: dict[str, int] = {} - for index, field in enumerate(self.fields): - if field.name == "": - found.extend( - problem( - ("fields", index, "name"), - "expected a non-empty field name", - "invalid_value", - ) - ) - first = seen.setdefault(field.name, index) - if first != index: - found.extend( - problem( - ("fields", index, "name"), - f"duplicate field name {field.name!r}, already used by field {first}", - "invalid_value", - ) - ) - if ( - isinstance(field.data_type, DataTypeEntity) - and field.data_type.storage_class() == "variable_length" - ): - found.extend( - problem( - ("fields", index, "data_type"), - "struct fields must use fixed-size data types", - "invalid_value", - ) - ) - if len(found) != 0: - raise MetadataValidationError(found) + problems = struct_problems def canonical(self) -> Self: """Each field's data type in its own canonical form.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 5532e0b946..d9e665c9ad 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -40,9 +40,11 @@ kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity` or `StorageTransformerEntity`; declare the configuration as dataclass -fields; put every rule finer than a type in `__post_init__`; add the -class to a scope. Complete, and runnable as written: +fields; write every rule finer than a type as a function of the +instance that yields problems, and bind it as `problems`; add the class +to a scope. Complete, and runnable as written: + from collections.abc import Iterator from dataclasses import dataclass from typing import ClassVar, Literal, NotRequired @@ -53,8 +55,7 @@ class to a scope. Complete, and runnable as written: CORE_AND_EXTENSIONS, UNSET, BytesBytesCodec, - MetadataValidationError, - problem, + ValidationProblem, ) class AcmeLz4Configuration(TypedDict, closed=True): @@ -64,21 +65,20 @@ class AcmeLz4Object(TypedDict, closed=True): name: Literal["acme.lz4"] configuration: AcmeLz4Configuration - @dataclass(frozen=True) # load-bearing: `coerce` builds the entity with cls(**members) + def acme_lz4_problems(codec: "AcmeLz4Codec", /) -> Iterator[ValidationProblem]: + if codec.acceleration is not UNSET and not 1 <= codec.acceleration <= 65537: + yield ValidationProblem( + ("acceleration",), + f"expected an integer in [1, 65537], got {codec.acceleration}", + "invalid_value", + ) + + @dataclass(frozen=True) # the fields are the schema; frozen, so an entity is a value class AcmeLz4Codec(BytesBytesCodec): - acceleration: int | UNSET = UNSET # optional: defaults to UNSET, never to a value + acceleration: int | UNSET = UNSET # optional: absent reads as UNSET identifier: ClassVar[str] = "acme.lz4" - - def __post_init__(self) -> None: - if self.acceleration is not UNSET and not 1 <= self.acceleration <= 65537: - raise MetadataValidationError( - problem( - ("acceleration",), - f"expected an integer in [1, 65537], got {self.acceleration}", - "invalid_value", - ) - ) + problems = acme_lz4_problems def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: if self.acceleration is UNSET: @@ -104,14 +104,15 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: absent is read that way where it is used, not defaulted. Everything finer than a type -- a bound, a rule about one member, members -read together -- is `__post_init__`, in plain code. It collects every -problem it finds and raises once; `coerce` catches the same error and -reports the problems in the document instead. `problem(loc, message, -kind)` returns a *one-element tuple*, so several are collected with -`found.extend(problem(...))` and raised as `MetadataValidationError(found)`; -pass `kind="invalid_value"` for a value rule, since the default names a -type mismatch. `__post_init__` runs only on an entity whose members all -read: a member of the wrong type is reported and the entity is not built. +read together -- is a function of the instance that yields +`ValidationProblem(loc, message, kind)` as it finds each, in plain +code, bound on the class as `problems`. Locations are relative to the +configuration, and `kind` is `"invalid_value"` for a value rule. The +constructor stops at the first problem it yields, so +`AcmeLz4Codec(acceleration=0)` raises `MetadataValidationError`; +`coerce` runs it to the end and reports every problem in the document. +It runs only on an entity whose members all read: a member of the wrong +type is reported and the entity is not built. **What an entity answers for itself**, beyond its fields. `to_json`, abstract: the entity as a document writes it, as a literal of its own diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 07e6aca0b2..6b720f36cd 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, replace -from typing import ClassVar, Literal, NotRequired, Self +from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, Self import pytest from typing_extensions import TypedDict @@ -36,6 +36,9 @@ written, ) +if TYPE_CHECKING: + from collections.abc import Iterator + class AcmeAffineConfiguration(TypedDict, closed=True): scale: float @@ -49,6 +52,17 @@ class AcmeAffineObject(TypedDict, closed=True): must_understand: NotRequired[bool] +def acme_affine_problems(codec: AcmeAffineCodec, /) -> Iterator[ValidationProblem]: + if codec.scale == 0: + yield ValidationProblem(("scale",), "expected a non-zero number, got 0", "invalid_value") + if isinstance(codec.dtype, DataTypeEntity) and codec.dtype.storage_class() == "variable_length": + yield ValidationProblem( + ("dtype",), + f"expected a fixed-size data type, got {type(codec.dtype).identifier!r}", + "invalid_value", + ) + + @dataclass(frozen=True) class AcmeAffineCodec(ArrayArrayCodec): """`x * scale + offset`, stored as `dtype` if one is named.""" @@ -58,24 +72,7 @@ class AcmeAffineCodec(ArrayArrayCodec): dtype: DataTypeEntity | Opaque | UNSET = UNSET identifier: ClassVar[str] = "acme.affine" - - def __post_init__(self) -> None: - found: list[ValidationProblem] = [] - if self.scale == 0: - found.extend(problem(("scale",), "expected a non-zero number, got 0", "invalid_value")) - if ( - isinstance(self.dtype, DataTypeEntity) - and self.dtype.storage_class() == "variable_length" - ): - found.extend( - problem( - ("dtype",), - f"expected a fixed-size data type, got {type(self.dtype).identifier!r}", - "invalid_value", - ) - ) - if len(found) != 0: - raise MetadataValidationError(found) + problems = acme_affine_problems def canonical(self) -> Self: """An offset of 0 is the identity, and absent says the same; `dtype` in its own form.""" diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 39fb1116b2..96c08b3130 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -15,7 +15,7 @@ from typing_extensions import ReadOnly, TypedDict if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Iterable, Iterator from zarr_metadata.v3.entity import ( @@ -70,6 +70,25 @@ class AcmeDecimal(TypedDict, closed=True): ] +def acme_decimal_problems(data_type: AcmeDecimalDataType, /) -> Iterator[ValidationProblem]: + if not 1 <= data_type.precision <= ACME_DECIMAL_MAX_PRECISION: + yield ValidationProblem( + ("precision",), + f"expected an integer in [1, {ACME_DECIMAL_MAX_PRECISION}], got {data_type.precision}", + "invalid_value", + ) + if data_type.scale < 0: + yield ValidationProblem( + ("scale",), f"expected an integer >= 0, got {data_type.scale}", "invalid_value" + ) + elif data_type.scale > data_type.precision: + yield ValidationProblem( + ("scale",), + f"expected an integer <= precision ({data_type.precision}), got {data_type.scale}", + "invalid_value", + ) + + @dataclass(frozen=True) class AcmeDecimalDataType(DataTypeEntity): """The `acme.decimal` data type, coerced from its metadata.""" @@ -79,32 +98,7 @@ class AcmeDecimalDataType(DataTypeEntity): identifier: ClassVar[str] = ACME_DECIMAL_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "multi_byte" - - def __post_init__(self) -> None: - found: list[ValidationProblem] = [] - if not 1 <= self.precision <= ACME_DECIMAL_MAX_PRECISION: - found.extend( - problem( - ("precision",), - f"expected an integer in [1, {ACME_DECIMAL_MAX_PRECISION}], " - f"got {self.precision}", - "invalid_value", - ) - ) - if self.scale < 0: - found.extend( - problem(("scale",), f"expected an integer >= 0, got {self.scale}", "invalid_value") - ) - elif self.scale > self.precision: - found.extend( - problem( - ("scale",), - f"expected an integer <= precision ({self.precision}), got {self.scale}", - "invalid_value", - ) - ) - if len(found) != 0: - raise MetadataValidationError(found) + problems = acme_decimal_problems def to_json(self) -> AcmeDecimal: return { @@ -320,10 +314,14 @@ def test_error_scale_above_precision() -> None: ] -def test_error_every_member_problem_is_reported_at_once() -> None: +def test_error_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: with pytest.raises(MetadataValidationError) as caught: AcmeDecimalDataType(precision=0, scale=-1) - assert _locs(caught.value.problems) == [("precision",), ("scale",)] + assert _locs(caught.value.problems) == [("precision",)] + _, problems = SCOPE.coerce( + DataTypeEntity, {"name": "acme.decimal", "configuration": {"precision": 0, "scale": -1}} + ) + assert _locs(problems) == [("configuration", "precision"), ("configuration", "scale")] def test_error_fill_value_must_be_a_string() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 4a20de3b28..5bcac47b70 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,7 +8,7 @@ import re from dataclasses import dataclass, replace -from typing import Annotated, ClassVar, Literal, NotRequired, Self, cast +from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self, cast import pytest from typing_extensions import TypedDict @@ -45,9 +45,21 @@ written, ) +if TYPE_CHECKING: + from collections.abc import Iterator + ACME_MAX_ACCELERATION = 65537 +def acme_lz4_problems(codec: AcmeLz4Codec, /) -> Iterator[ValidationProblem]: + if codec.acceleration is not UNSET and not 1 <= codec.acceleration <= ACME_MAX_ACCELERATION: + yield ValidationProblem( + ("acceleration",), + f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {codec.acceleration}", + "invalid_value", + ) + + @dataclass(frozen=True) class AcmeLz4Codec(BytesBytesCodec): """A third-party compressor.""" @@ -56,16 +68,7 @@ class AcmeLz4Codec(BytesBytesCodec): identifier: ClassVar[str] = "acme.lz4" variable_size: ClassVar[bool] = True - - def __post_init__(self) -> None: - if self.acceleration is not UNSET and not 1 <= self.acceleration <= ACME_MAX_ACCELERATION: - raise MetadataValidationError( - problem( - ("acceleration",), - f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {self.acceleration}", - "invalid_value", - ) - ) + problems = acme_lz4_problems def to_json(self) -> ZarrV3MetadataFieldJSON: if self.acceleration is UNSET: @@ -326,8 +329,8 @@ def test_a_third_party_can_register_a_family() -> None: def test_error_a_member_needs_a_check_from_somewhere() -> None: - # An annotation outside the shapes `check_for` compiles implies no - # check, so the entity owes one. Silently skipping the member would + # An annotation outside the shapes the parser reads implies no + # parser, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): @@ -477,7 +480,14 @@ class AcmeBlockObject(TypedDict, closed=True): must_understand: NotRequired[bool] -# A third-party rule about a member, written in `__post_init__`. +# A third-party rule about a member: a function of the instance. +def acme_block_problems(codec: AcmeBlockCodec, /) -> Iterator[ValidationProblem]: + if codec.block < 1 or codec.block & (codec.block - 1) != 0: + yield ValidationProblem( + ("block",), f"expected a power of two, got {codec.block}", "invalid_value" + ) + + @dataclass(frozen=True) class AcmeBlockCodec(BytesBytesCodec): """A codec whose block size must be a power of two.""" @@ -485,21 +495,16 @@ class AcmeBlockCodec(BytesBytesCodec): block: int identifier: ClassVar[str] = "acme.block" - - def __post_init__(self) -> None: - if self.block < 1 or self.block & (self.block - 1) != 0: - raise MetadataValidationError( - problem(("block",), f"expected a power of two, got {self.block}", "invalid_value") - ) + problems = acme_block_problems def to_json(self) -> ZarrV3MetadataFieldJSON: return {"name": "acme.block", "configuration": {"block": self.block}} -def test_a_rule_about_a_member_is_post_init() -> None: +def test_a_rule_about_a_member_is_a_function_of_the_instance() -> None: # The rule runs on the typed members and reports relative to the - # configuration; `coerce` catches what it raises and locates it in - # the document, and the constructor raises it as it is. + # configuration; `coerce` runs it to the end and locates what it + # yields in the document; the constructor stops at the first. scope = CORE_AND_EXTENSIONS.extended_with(AcmeBlockCodec) codec, problems = scope.coerce( CodecEntity, {"name": "acme.block", "configuration": {"block": 64}} @@ -519,6 +524,59 @@ def test_a_rule_about_a_member_is_post_init() -> None: assert [p.kind for p in problems] == ["invalid_type"] +def acme_range_problems(codec: AcmeRangeCodec, /) -> Iterator[ValidationProblem]: + if codec.low < 0: + yield ValidationProblem( + ("low",), f"expected an integer >= 0, got {codec.low}", "invalid_value" + ) + if codec.high < codec.low: + yield ValidationProblem( + ("high",), f"expected an integer >= low, got {codec.high}", "invalid_value" + ) + + +@dataclass(frozen=True) +class AcmeRangeCodec(BytesBytesCodec): + """A codec with two rules, so that one can fail after another.""" + + low: int + high: int + + identifier: ClassVar[str] = "acme.range" + problems = acme_range_problems + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return {"name": "acme.range", "configuration": {"low": self.low, "high": self.high}} + + +def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: + # One function, two consumers: the constructor takes the first + # problem it yields, `coerce` runs it to the end. A consumer holding + # an entity may run it too, and stop or collect as it likes. + with pytest.raises(MetadataValidationError) as caught: + AcmeRangeCodec(low=-1, high=-2) + assert [p.loc for p in caught.value.problems] == [("low",)] + scope = CORE_AND_EXTENSIONS.extended_with(AcmeRangeCodec) + _, problems = scope.coerce( + CodecEntity, {"name": "acme.range", "configuration": {"low": -1, "high": -2}} + ) + assert [p.loc for p in problems] == [("configuration", "low"), ("configuration", "high")] + assert list(acme_range_problems(AcmeRangeCodec(low=0, high=1))) == [] + + +def test_error_an_entity_may_not_define_post_init() -> None: + # `coerce` never runs it, so a rule written there would judge a + # hand-built entity and no document. + with pytest.raises(TypeError, match="defines __post_init__; write its rules as a function"): + + @dataclass(frozen=True) + class Checked(BytesBytesCodec): + identifier: ClassVar[str] = "acme.checked" + + def __post_init__(self) -> None: + return None + + def test_a_slotted_entity_is_accepted() -> None: # `@dataclass(slots=True)` builds the class twice, so class creation # sees it twice; the second time its members are slot descriptors. From 6e5053daa0a27ee3bb5d0f2caca06f5fd1e907ba Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 17:09:07 +0200 Subject: [PATCH 086/107] fix(zarr-metadata): what three reviews of the entity layer found Three reviews of the rebuilt layer, each from the vantage of another implementation (zarrs, TensorStore, zarrita.js), agreed on a short list of defects and complexity; this is the part that needed no decision. Every codec declares `variable_size`. The default of `False` was a verdict: the door's own example compressor, which set nothing, was accepted as a shard-index codec, where `gzip` is refused. Registration is the one moment an entity is refused. `__init_subclass__` and the twelve `base=True` flags are gone; `_registrable` asks `unreadable`, which names a field whose annotation is not a shape JSON takes (a type defined inside a function among them, now a message rather than a bare `NameError`), a `__post_init__` of the entity's own, and an owed class variable left unset, before the abstract methods. Nothing happens at class creation, so a family is a plain subclass and a forward reference resolves. The envelope is judged once. `Context.coerce` runs the metadata-field check, reports it only when the model layer has not, and does not read an entity whose value names none or whose configuration is not an object; the entity no longer adds "expected a metadata field" beside the model's report of the same defect, nor "requires a configuration" beside "expected a mapping". `ArrayDocumentV3.from_json` reads whatever the structure allowed, so a structural problem no longer hides the semantic ones. A rule about the member the envelope's name carries lands on the field, not under a `configuration` the document does not have. A name resolves by asking each entity of the kind whether it is its own, in registration order; the invented-identifier veto is gone. `Opaque` answers `to_json` and `canonical` -- the JSON it kept, and itself -- so `written` and `canonicalized` are gone and a field typed `CodecEntity | Opaque` is written and simplified without asking which it holds. In the parser: `Literal[1]` refuses JSON `true` (`True == 1` in Python); a `Literal` of mixed types sorts by repr instead of failing on a comparison; a fixed-length array accepts a list as a homogeneous one does; a key inside a nested object is reported at the key, as a top-level one is; a record dataclass that refuses its own values is reported, located under the object, rather than raised out of `coerce`. Over the 40,000-document corpus, no verdict changes. Lost: 18,067 duplicate reports of a malformed envelope, 1,263 "requires a configuration" beside "expected a mapping", 340 judgments of an entity whose configuration was not an object (it was judged as having none), and 42 nested-key locations, gained back at the key. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.bugfix.md | 9 +- .../zarr-metadata/changes/4379.feature.10.md | 11 +- .../zarr-metadata/changes/4379.feature.7.md | 10 +- packages/zarr-metadata/changes/4379.misc.2.md | 46 +++- .../src/zarr_metadata/v3/_document.py | 31 ++- .../src/zarr_metadata/v3/_entity.py | 214 +++++++++--------- .../src/zarr_metadata/v3/_registry.py | 101 ++++----- .../src/zarr_metadata/v3/_typed_json.py | 43 ++-- .../src/zarr_metadata/v3/codec/bytes.py | 1 + .../src/zarr_metadata/v3/codec/cast_value.py | 7 +- .../src/zarr_metadata/v3/codec/crc32c.py | 1 + .../zarr_metadata/v3/codec/scale_offset.py | 1 + .../v3/codec/sharding_indexed.py | 10 +- .../src/zarr_metadata/v3/codec/transpose.py | 1 + .../zarr_metadata/v3/data_type/_families.py | 8 +- .../src/zarr_metadata/v3/data_type/struct.py | 6 +- .../src/zarr_metadata/v3/entity.py | 53 ++--- .../tests/v3/test_acme_affine.py | 7 +- .../tests/v3/test_extension_api.py | 184 +++++++++++---- 19 files changed, 446 insertions(+), 298 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.bugfix.md b/packages/zarr-metadata/changes/4379.bugfix.md index 756cc6baf4..b302942838 100644 --- a/packages/zarr-metadata/changes/4379.bugfix.md +++ b/packages/zarr-metadata/changes/4379.bugfix.md @@ -13,11 +13,10 @@ rules layer: or inner chunk shape went unreported. Every chunk of an array has the array's rank whatever the grid, so the rank is kept and only the geometry checks stand down. -- One unusable configuration member suppressed every other judgment - about the same entity — a misspelled `index_location` hid the problems - in a shard's inner pipelines. An optional member that fails its type - check now falls back to absent, so the rest of the entity is still - judged. +- One unusable configuration member suppressed every judgment about the + entities nested inside the same entity — a misspelled `index_location` + hid the problems in a shard's inner pipelines. A nested entity is now + read whatever else was found, so its problems are reported alongside. - Problems about an entity as a whole pointed at a `configuration` node that a bare-string entity does not have (`("codecs", 0, "configuration")` for `"codecs": ["bytes"]`). They now point at the diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index b65416a179..86e9a4db12 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -37,8 +37,9 @@ valid under both reports identically. Among those invalid under both, 4,397 report fewer problems and 15,259 report more, the latter because a problem that used to stand down the rest of an entity no longer does. -Two ways of writing an entity type-check cleanly and then fail somewhere -that will not name the class, so class creation refuses them: a field -whose annotation is not a shape JSON takes, and a class variable the -entity owes and did not declare (read off the annotations, so a family -adding one cannot forget to require it). +Some ways of writing an entity type-check cleanly and then fail +somewhere that will not name the class, so registration refuses them: a +field whose annotation is not a shape JSON takes, a `__post_init__` of +the entity's own, and a class variable the entity owes and did not +declare (read off the annotations, so a family adding one cannot forget +to require it). diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index b47bef257b..2c920f1ea7 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -37,7 +37,7 @@ dataclass, an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested metadata field, written as its kind with `Opaque`. That covers every member this package models; an annotation outside them is refused -at class creation, and `Annotated[str, FROM_NAME]` marks the one field +at registration, and `Annotated[str, FROM_NAME]` marks the one field carried by the envelope's name rather than a configuration key (`r`), which `coerce` fills from the envelope. Whether a configuration is required follows too, since the spec ties the bare-name spelling to whether any member is @@ -46,8 +46,8 @@ required. Nothing is derived from the fields ahead of time: `coerce` parses a configuration against them as it reads it. Field annotations are resolved per class, skipping class variables by text, so a `ClassVar` -naming something imported only for the type checker cannot fail class -creation. +naming something imported only for the type checker cannot fail +registration. One diagnostic became more precise: a malformed `[value, count]` pair in a rectilinear grid is now reported at the offending element inside the @@ -65,7 +65,7 @@ are the containing entity's own two lines, `written(self.inner)` in spellings of its members mean the same -- a rectilinear dimension's run-length encoding, a `typesize` that `noshuffle` ignores. A field typed as bare `MetadataEntity`, or as an entity without `Opaque`, is -refused at class creation: no scope could place the one, and the other +refused at registration: no scope could place the one, and the other lies about what the field holds when the inner name is out of scope. Everything finer than a type -- a bound, a rule about one member, a @@ -81,7 +81,7 @@ place per entity is what will show where a shared form is worth having. The set of annotation shapes the parser reads is closed: the shapes JSON takes, and no others. A field annotation outside them is refused -at class creation, and the field is written as one of them instead, with +at registration, and the field is written as one of them instead, with any finer rule in the function bound as `problems`. The parser is one module that knows nothing of entities; the entity layer hands it the one shape of its own, a field holding an entity, as a leaf it reads at any depth. diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 5100ffb04f..dfdad2673b 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -30,15 +30,17 @@ package's own parser as its oracle, it reads two shapes it did not: type it names -- which also closes a gap for a third-party field typed with either. -Class creation refuses two things and no more: a field whose annotation -is not a shape JSON takes, and a class variable a base annotates and -nothing sets. Everything else an author could get wrong, pyright says -in the editor -- a field shadowing a class variable, a `Literal` class -variable outside its values -- or registration says, with what to -write: no `@dataclass`, a kind's abstract method left undefined, a -codec subclassing `CodecEntity` instead of a kind. Nothing is kept on -the class: what the layer needs of an entity's fields, it reads off -them when it reads a document. +Registration is the one moment an entity is refused, with what to +write: no `@dataclass`, a codec subclassing `CodecEntity` instead of a +kind, a field whose annotation is not a shape JSON takes, a +`__post_init__` of the entity's own, a class variable a base annotates +and nothing sets, a kind's abstract method left undefined. Nothing +happens at class creation, so a family is a plain subclass and a +forward reference in a field annotation resolves. Everything else an +author could get wrong, pyright says in the editor -- a field shadowing +a class variable, a `Literal` class variable outside its values. +Nothing is kept on the class: what the layer needs of an entity's +fields, it reads off them when it reads a document. Four adversarial reviews -- two extension authors writing a codec and a data type against the door alone, a design review, an onboarding review @@ -77,8 +79,10 @@ annotations exists. `canonical` is the entity's own, as `to_json` is: the entity itself by default, overridden where two spellings mean the same and, in an entity -that contains entities, to put those in canonical form with -`canonicalized`. One diagnostic is more complete: when one element of a +that contains entities, to put those in canonical form; an `Opaque` +answers both, with the JSON it kept and with itself, so a field typed +`CodecEntity | Opaque` is written and simplified without asking which +it holds. One diagnostic is more complete: when one element of a shard's pipeline is not a metadata field at all, the other elements are still read and judged, where before the whole member stood down. Over the 40,000-document corpus that adds 133 problems to 91 documents @@ -109,3 +113,23 @@ wanted: it reads every TypedDict as `Mapping[str, object]`, never as the python/mypy#18439 -- mypy lacks PEP 728). The conversion is sound and the annotation stays; a consumer under mypy casts at the one place it puts an entity's JSON into a document. + +From three reviews of the result, each from the vantage of another +implementation (zarrs, TensorStore, zarrita.js): every codec declares +`variable_size`, because a default in either direction is a verdict +and the door's own example compressor was passing as a shard-index +codec; a malformed envelope is one problem, judged once by the scope, +where the entity used to add a second at the same location (a +configuration that is not an object no longer also reports "requires a +configuration"; a value that is not a metadata field no longer also +reports "expected a metadata field"); `ArrayDocumentV3.from_json` +reports structural and semantic problems together instead of hiding +the second behind the first; a problem about the member the envelope's +name carries lands on the field, not under a `configuration` the +document does not have; a name resolves by asking each entity of the +kind whether it is its own, so the invented-identifier veto is gone; +`Literal[1]` refuses JSON `true`, a `Literal` of mixed types no longer +fails on a comparison, a fixed-length array accepts a list as a +homogeneous one does, a key inside a nested object is reported at the +key like a top-level one, and a record dataclass that refuses its own +values is reported rather than raised. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index a27985fe02..7a1ac22e96 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -37,9 +37,7 @@ MetadataEntity, Opaque, StorageTransformerEntity, - canonicalized, within, - written, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context @@ -103,11 +101,11 @@ def canonical(self) -> ArrayDocumentV3: return replace( self, document=document, - data_type=canonicalized(self.data_type), - chunk_grid=canonicalized(self.chunk_grid), - chunk_key_encoding=canonicalized(self.chunk_key_encoding), - codecs=tuple(canonicalized(codec) for codec in self.codecs), - storage_transformers=tuple(canonicalized(entry) for entry in self.storage_transformers), + data_type=self.data_type.canonical(), + chunk_grid=self.chunk_grid.canonical(), + chunk_key_encoding=self.chunk_key_encoding.canonical(), + codecs=tuple(codec.canonical() for codec in self.codecs), + storage_transformers=tuple(entry.canonical() for entry in self.storage_transformers), ) def to_json(self) -> dict[str, object]: @@ -119,11 +117,11 @@ def to_json(self) -> dict[str, object]: Ask `canonical` first for the simplest equivalent spelling. """ rendered: dict[str, object] = { - "data_type": written(self.data_type), - "chunk_grid": written(self.chunk_grid), - "chunk_key_encoding": written(self.chunk_key_encoding), - "codecs": tuple(written(codec) for codec in self.codecs), - "storage_transformers": tuple(written(entry) for entry in self.storage_transformers), + "data_type": self.data_type.to_json(), + "chunk_grid": self.chunk_grid.to_json(), + "chunk_key_encoding": self.chunk_key_encoding.to_json(), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "storage_transformers": tuple(entry.to_json() for entry in self.storage_transformers), } return { **self.document, @@ -148,10 +146,11 @@ def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> """ normalized = arrays_to_tuples(value) problems = validate_array_metadata_v3_structure(normalized) - if isinstance(normalized, Mapping) and len(problems) == 0: - document = cast("Mapping[str, object]", normalized) - array, found = read_array_v3(document, context) - problems = (*found, *array.problems()) + if isinstance(normalized, Mapping): + # Read whatever the structure allowed, so a structural + # problem does not hide the semantic ones behind it. + array, found = read_array_v3(cast("Mapping[str, object]", normalized), context) + problems = (*problems, *found, *array.problems()) if len(problems) == 0: return array if len(problems) == 0: # pragma: no cover - a non-mapping always has problems diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 362e208959..7cc2fce406 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -21,7 +21,8 @@ entity's to answer through `incoming_problems`, `shape_problems`, `fill_value_problems`, `transition` and `grid`, each taking the part of the document it needs. The document that composes those answers is -`_document`; which entities are in scope is `_registry`. +`_document`; which entities are in scope is `_registry`, which refuses, +at registration, an entity `coerce` could not read. """ from __future__ import annotations @@ -132,28 +133,29 @@ def within(prefix: Loc, problems: Sequence[ValidationProblem]) -> tuple[Validati def named_configuration( value: object, -) -> tuple[str | None, Mapping[str, object] | None, bool]: - """Split metadata into `(name, configuration, must_understand)`. +) -> tuple[str | None, Mapping[str, object] | None, tuple[ValidationProblem, ...]]: + """Split metadata into `(name, configuration, problems)`. The shared shape every entity arrives in: a bare name, or an object - carrying one. A `None` name means the value is not a metadata field at - all; a `None` configuration means the bare spelling was used. + carrying one. A `None` name means the value is not a metadata field + at all; a `None` configuration means the bare spelling was used, or + the key was left out. A configuration that is present and not an + object is the one problem reported, at `("configuration",)`. """ if isinstance(value, str): - return value, None, True + return value, None, () if not isinstance(value, Mapping): - return None, None, True + return None, None, () entry = cast("Mapping[str, object]", value) name = entry.get("name") if not isinstance(name, str): - return None, None, True - configuration = entry.get("configuration") - must_understand = entry.get("must_understand", True) - return ( - name, - cast("Mapping[str, object]", configuration) if isinstance(configuration, Mapping) else None, - must_understand if isinstance(must_understand, bool) else True, - ) + return None, None, () + if "configuration" not in entry: + return name, None, () + configuration = entry["configuration"] + if not isinstance(configuration, Mapping): + return name, None, problem(("configuration",), f"expected an object, got {configuration!r}") + return name, cast("Mapping[str, object]", configuration), () def is_metadata_field(value: object, loc: Loc) -> tuple[ValidationProblem, ...]: @@ -179,30 +181,78 @@ class Opaque: model, which is not an error and is the reader's cue to resolve it elsewhere. `invalid` is a name that *was* claimed and then refused, for the reasons reported alongside. + + Answers `to_json` and `canonical` as an entity does, so a field typed + `CodecEntity | Opaque` is written and simplified without asking + which case it holds. """ json: object reason: Literal["out_of_scope", "invalid"] + def to_json(self) -> ZarrV3MetadataFieldJSON: + """The JSON the document wrote, as it wrote it. -def written(value: MetadataEntity | Opaque) -> ZarrV3MetadataFieldJSON: - """A contained metadata field as a document would write it. + An `Opaque` inside a built entity is out of scope -- an inner + name no entity in scope claimed -- and its JSON passed the + envelope check as a metadata field, which is what the cast says. + """ + return cast("ZarrV3MetadataFieldJSON", self.json) - The entity's own JSON, or the JSON an `Opaque` kept. An `Opaque` - inside a built entity is out of scope -- an inner name no entity in - scope claimed -- and its JSON passed the envelope check as a metadata - field, which is what the cast says. - """ - if isinstance(value, MetadataEntity): - return value.to_json() - return cast("ZarrV3MetadataFieldJSON", value.json) + def canonical(self) -> Self: + """Itself: what was not read cannot be simplified.""" + return self -def canonicalized(value: EntityT | Opaque) -> EntityT | Opaque: - """A contained metadata field in canonical form: the entity's own, or the `Opaque` as it is.""" - if isinstance(value, MetadataEntity): - return value.canonical() - return value +def unreadable(cls: type[MetadataEntity]) -> str | None: + """Why `coerce` could not read an instance of `cls`; None if it can. + + Three things type-check cleanly and then go wrong somewhere that + will not name the class: a field whose annotation is not a shape + JSON takes, which `coerce` could not parse; a `__post_init__` of the + entity's own, whose rules `coerce` would never ask; and a class + variable a base annotates and nothing sets -- `identifier` for every + entity, `bounds` for an integer type -- which the first lookup would + fail. Registration asks, and refuses the class with the answer. + """ + try: + hints = field_hints(cls) + except NameError as unresolved: + return ( + f"{cls.__name__}: a field annotation names {unresolved.name!r}, which is not " + "defined where the class is; define it at module level, or import it outside " + "`TYPE_CHECKING`" + ) + unread: list[str] = [] + for name, annotation in hints.items(): + try: + accepted = parser_for(annotation, _reading(None, [])) is not None + except TypeError as refused: + return f"{cls.__name__}: {name} {refused}" + if not accepted: + unread.append(name) + if len(unread) != 0: + return ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in unread)}" + ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in the function bound as `problems`" + ) + if "__post_init__" in vars(cls): + return ( + f"{cls.__name__} defines __post_init__; write its rules as a function of the " + "instance that yields problems and bind it as `problems = `: the " + "constructor stops at the first problem it yields, `coerce` reports every one" + ) + annotated = declared_class_vars(cls) + missing = sorted(name for name in annotated if not hasattr(cls, name)) + if len(missing) != 0: + owed = ", ".join(f"{name} (annotated by {annotated[name].__name__})" for name in missing) + return f"{cls.__name__} does not declare {owed}; set each as a class variable" + return None def nested_kind(annotation: object) -> type[MetadataEntity] | None: @@ -212,7 +262,7 @@ def nested_kind(annotation: object) -> type[MetadataEntity] | None: because that is what the field holds when the inner name is out of scope, and a kind -- or a subclass of one, `GzipCodec` -- because a scope resolves names by kind. An annotation naming an entity any - other way is a `TypeError` saying so, which class creation reports + other way is a `TypeError` saying so, which registration reports against the field. """ parts = get_args(annotation) if is_union(annotation) else (annotation,) @@ -242,7 +292,7 @@ def _reading(context: Context | None, nested: list[ValidationProblem]) -> Leaf: entity is collected in `nested`, apart, because the containing entity's rules still run over its own members when only a contained entity is wrong. With no `context` the shape is checked and nothing - is read, which is what class creation asks. + is read, which is what registration asks. """ def leaf(annotation: object) -> Parser | None: @@ -299,64 +349,6 @@ class MetadataEntity(ABC): an invented identifier that no real name can collide with. """ - def __init_subclass__(cls, *, base: bool = False, **kwargs: object) -> None: - """Refuse, at class creation, an entity this layer could not read. - - Three things type-check cleanly and then go wrong somewhere that - will not name the class: a field whose annotation is not a shape - JSON takes, which `coerce` could not parse; a `__post_init__` of - the entity's own, whose rules `coerce` would never ask; and a - class variable a base annotates and nothing sets -- `identifier` - for every entity, `bounds` for an integer type -- which the first - lookup would fail. An import-time error in the extension's own - module is the one place the author is looking. - - `base=True` for a class that exists to add a class variable - rather than to be an entity -- `CodecEntity`, `IntegerDataType`. - """ - super().__init_subclass__(**kwargs) - if base: - return - hints = field_hints(cls) - unread: list[str] = [] - for name, annotation in hints.items(): - try: - accepted = parser_for(annotation, _reading(None, [])) is not None - except TypeError as refused: - msg = f"{cls.__name__}: {name} {refused}" - raise TypeError(msg) from None - if not accepted: - unread.append(name) - if len(unread) != 0: - msg = ( - f"{cls.__name__}: " - f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in unread)}" - ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " - "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " - "record, Mapping[str, V], a NewType, or an entity kind with Opaque " - "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in the function bound as `problems`" - ) - raise TypeError(msg) - if "__post_init__" in vars(cls): - msg = ( - f"{cls.__name__} defines __post_init__; write its rules as a function of the " - "instance that yields problems and bind it as `problems = `: the " - "constructor stops at the first problem it yields, `coerce` reports every one" - ) - raise TypeError(msg) - annotated = declared_class_vars(cls) - missing = sorted(name for name in annotated if not hasattr(cls, name)) - if len(missing) != 0: - owed = ", ".join( - f"{name} (annotated by {annotated[name].__name__})" for name in missing - ) - msg = ( - f"{cls.__name__} does not declare {owed}; set each as a class variable, " - "or pass base=True if this class exists only to be subclassed" - ) - raise TypeError(msg) - def problems(self, /) -> Iterator[ValidationProblem]: """Every reason this entity's values are not allowed, yielded as found. @@ -409,9 +401,11 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: configuration -- and handed back only when everything inside it read too. """ - name, given, _ = named_configuration(value) + name, given, envelope = named_configuration(value) if name is None or not cls.accepts(name): return None, problem((), f"expected the {cls.identifier!r} entity") + if len(envelope) != 0: + return None, envelope hints = field_hints(cls) if given is None and any( not is_from_name(annotation) and not is_optional(annotation) @@ -458,7 +452,19 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # read is a hole, and judging around it would be guessing. return None, found entity = cls._unchecked(members) - refused = within((), tuple(entity.problems())) + # A problem about a member the envelope's name carries is about + # the entity, and lands on it rather than under a configuration + # the document does not have. + from_name = {key for key, annotation in hints.items() if is_from_name(annotation)} + refused = within( + (), + tuple( + ValidationProblem((), found.message, found.kind) + if len(found.loc) != 0 and found.loc[0] in from_name + else found + for found in entity.problems() + ), + ) if len(refused) != 0: # Values the spec disallows: reported rather than raised, # every one, located under the configuration. @@ -511,7 +517,7 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: @dataclass(frozen=True) -class CodecEntity(MetadataEntity, base=True): +class CodecEntity(MetadataEntity): """An entity that occupies a position in the codec pipeline. Of one of three kinds, each a base class: `ArrayArrayCodec`, @@ -519,11 +525,12 @@ class CodecEntity(MetadataEntity, base=True): pipeline the codec may stand, and what it must answer. """ - variable_size: ClassVar[bool] = False + variable_size: ClassVar[bool] """Whether this codec's output size depends on the bytes it is given. A compressor's does, so a shard index encoded with one has no size - derivable from metadata alone, and the shard cannot be read. + derivable from metadata alone, and the shard cannot be read. Every + codec says, because a default in either direction is a verdict. """ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: @@ -538,7 +545,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb @dataclass(frozen=True) -class ArrayArrayCodec(CodecEntity, base=True): +class ArrayArrayCodec(CodecEntity): """A codec that transforms the array: what reaches the next codec is its to say.""" @abstractmethod @@ -553,17 +560,17 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: @dataclass(frozen=True) -class ArrayBytesCodec(CodecEntity, base=True): +class ArrayBytesCodec(CodecEntity): """The one codec in a pipeline that turns the array into bytes.""" @dataclass(frozen=True) -class BytesBytesCodec(CodecEntity, base=True): +class BytesBytesCodec(CodecEntity): """A codec that transforms bytes, after the array is gone.""" @dataclass(frozen=True) -class ChunkGridEntity(MetadataEntity, base=True): +class ChunkGridEntity(MetadataEntity): """An entity that divides an array into the parts a pipeline encodes.""" def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: @@ -585,7 +592,7 @@ def grid(self, array_shape: object) -> ChunkGrid: @dataclass(frozen=True) -class DataTypeEntity(MetadataEntity, base=True): +class DataTypeEntity(MetadataEntity): """An entity that says how the array's scalars are stored. Only data types answer that, and every rule that turns on it -- a @@ -614,12 +621,12 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class ChunkKeyEncodingEntity(MetadataEntity, base=True): +class ChunkKeyEncodingEntity(MetadataEntity): """An entity that says how a chunk's coordinates become a store key.""" @dataclass(frozen=True) -class StorageTransformerEntity(MetadataEntity, base=True): +class StorageTransformerEntity(MetadataEntity): """An entity that stands between the codec pipeline and the store.""" @@ -654,7 +661,6 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "Opaque", "StorageClass", "StorageTransformerEntity", - "canonicalized", "is_from_name", "is_integer", "is_metadata_field", @@ -662,6 +668,6 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "named_configuration", "nested_kind", "problem", + "unreadable", "within", - "written", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 5af57deb12..5e95b96af6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -13,11 +13,12 @@ `zarr-extensions` registers and this package models. A name in neither is not rejected — extension openness — it is simply not judged. -Identifiers are the `name` the metadata carries, with one exception. A -family covers many names with one class -- every `r` spelling is one -data-type family -- so it registers under an invented identifier that no -real name can collide with, and recognizes its own names through -`accepts`. +An entity is registered under its `identifier`, which is the `name` the +metadata carries -- except for a family, which covers many names with +one class (every `r` spelling is one data-type family) and registers +under an invented one. Resolution asks each entity of a kind whether a +name is its own, through `accepts`; the identifier is the key that +`extended_with` takes a name over with. """ from __future__ import annotations @@ -42,6 +43,7 @@ Opaque, kind_of, named_configuration, + unreadable, ) from zarr_metadata.v3._typed_json import is_class_var, own_annotations from zarr_metadata.v3.chunk_grid.rectilinear import RectilinearChunkGrid @@ -133,28 +135,17 @@ def resolve(self, kind: type[_EntityT], name: str) -> type[_EntityT] | None: an error: an unknown name may be an extension this reader does not model, and openness means leaving it unjudged. - The entity has the last word, via `accepts`. A name that is a key - still has to be claimed, because a family's key is an invented - identifier that no document may write; and a name that is not a - key may still belong to a family, which is what the scan is for. + Each entity of the kind is asked whether the name is its own, + through `accepts`, in registration order, and the first to claim + it answers for it. A family covers many names with one class, so + a table keyed by name could not hold it; the identifier keys + exist for `extended_with` to take a name over, not for lookup. """ registered = kind_of(kind) if registered is None: return None table = self.tables.get(registered, {}) - entity = table.get(name) - if entity is not None and not entity.accepts(name): - return None - if entity is None: - # A family covers many names with one class, so its entry - # cannot be keyed by all of them; it is keyed by an invented - # identifier and recognizes its own. Asked only when the name - # is not a key, so the common case stays a lookup. First - # match wins, and two entities claiming one name is a scope - # that contradicts itself. - entity = next( - (candidate for candidate in table.values() if candidate.accepts(name)), None - ) + entity = next((candidate for candidate in table.values() if candidate.accepts(name)), None) if entity is None or not issubclass(entity, kind): return None return entity @@ -182,43 +173,48 @@ def coerce( envelope gets the same structural judgment here that the model layer gives a top-level one -- an extra member, a `configuration` that is not an object, a `must_understand` that is not a boolean - or is `false`. That last one is why the flag is passed: an - extension point is something a reader must understand at every - depth, not only at the document's top level. - `envelope_judged` says that judgment has already happened, which - it has for the fields of a document the model layer accepted. + or is `false`. `envelope_judged` says the model layer has + reported that already, which it has for the fields of a + document, so it is not reported twice. The entity is read + whenever there is one to read -- a stray member or a malformed + `must_understand` says nothing about the configuration -- and + not when the value names no entity or its configuration is not + an object, which the envelope judgment has already said. """ - problems: list[ValidationProblem] = [] - if not envelope_judged: - problems.extend( + envelope = validate_metadata_field_v3(value, allow_must_understand_false=False) + problems = ( + () + if envelope_judged + else tuple( ValidationProblem((*loc, *found.loc), found.message, found.kind) - for found in validate_metadata_field_v3(value, allow_must_understand_false=False) - ) - name, _, _ = named_configuration(value) - if name is None: - return Opaque(value, "invalid"), ( - *problems, - ValidationProblem(loc, f"expected a metadata field, got {value!r}", "invalid_type"), + for found in envelope ) + ) + name, _, malformed = named_configuration(value) + if name is None or len(malformed) != 0: + return Opaque(value, "invalid"), problems entity_type = self.resolve(kind, name) if entity_type is None: - return Opaque(value, "out_of_scope"), tuple(problems) + return Opaque(value, "out_of_scope"), problems entity, found = entity_type.coerce(value, self) - problems.extend( - ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found + problems = ( + *problems, + *(ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found), ) if entity is None: - return Opaque(value, "invalid"), tuple(problems) - return entity, tuple(problems) + return Opaque(value, "invalid"), problems + return entity, problems def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: """The kind `entity` is registered under; `TypeError` for a class no scope can use. - Class creation refuses what it can see; these are the things it - cannot -- the decorator, what a kind leaves abstract, which base was - chosen -- checked at the first place the class passes through before - `coerce` builds it. + The one moment an entity is refused: every way of writing one that + type-checks cleanly and then fails somewhere that will not name the + class -- no kind, a codec skipping the kind classes, no `@dataclass`, + a field `coerce` could not parse, a `__post_init__` `coerce` would + never ask, a class variable owed and unset, what a kind leaves + abstract -- with a message that says what to write. """ kind = kind_of(entity) if kind is None: @@ -237,18 +233,21 @@ def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: "ArrayBytesCodec or BytesBytesCodec, which says what the codec does to the array" ) raise TypeError(msg) - if inspect.isabstract(entity): - left = ", ".join(sorted(entity.__abstractmethods__)) - msg = f"{entity.__name__} does not define {left}, which its base leaves abstract" - raise TypeError(msg) if "__dataclass_fields__" not in vars(entity) and any( not is_class_var(annotation) for annotation in own_annotations(entity).values() ): msg = ( f"{entity.__name__} declares fields but is not a dataclass; decorate it with " - "@dataclass(frozen=True), which is what `coerce` builds it with" + "@dataclass(frozen=True), which is what makes its fields the configuration" ) raise TypeError(msg) + refused = unreadable(entity) + if refused is not None: + raise TypeError(refused) + if inspect.isabstract(entity): + left = ", ".join(sorted(entity.__abstractmethods__)) + msg = f"{entity.__name__} does not define {left}, which its base leaves abstract" + raise TypeError(msg) return kind diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index 04715bb847..3b71a9eb8f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -44,7 +44,7 @@ from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import ValidationProblem, is_json +from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem, is_json if TYPE_CHECKING: from zarr_metadata.model._validation import ProblemKind @@ -201,7 +201,7 @@ def field_hints(cls: type) -> dict[str, object]: Each class's own annotations are resolved in that class's module, and class variables are skipped *before* resolving, by text -- so a `ClassVar` whose annotation names something imported only for the - type checker cannot fail class creation. `@dataclass` sees the same + type checker cannot fail registration. `@dataclass` sees the same set, in the same order. """ hints: dict[str, object] = {} @@ -337,11 +337,15 @@ def parse(value: object, loc: Loc) -> Parsed: _JSON: Parser = _scalar("a JSON value", is_json) -def one_of(allowed: tuple[str, ...]) -> Parser: - """A member whose type is a closed set of names.""" +def one_of(allowed: tuple[object, ...]) -> Parser: + """A member whose type is a closed set of values. + + Equal and of the same type: JSON `true` is not the integer 1, though + Python says `True == 1`. + """ def parse(value: object, loc: Loc) -> Parsed: - if value not in allowed: + if not any(value == entry and type(value) is type(entry) for entry in allowed): return value, problem( loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value" ) @@ -372,9 +376,9 @@ def fixed_tuple(elements: Sequence[Parser], description: str) -> Parser: """A member whose type is an array of a fixed length, parsed position by position.""" def parse(value: object, loc: Loc) -> Parsed: - if not isinstance(value, tuple): + if not isinstance(value, (list, tuple)): return value, problem(loc, f"expected {description}, got {value!r}") - entries = cast("tuple[object, ...]", value) + entries = tuple(cast("list[object] | tuple[object, ...]", value)) if len(entries) != len(elements): return entries, problem(loc, f"expected {description}, got {entries!r}") parsed: list[object] = [] @@ -427,18 +431,18 @@ def _keys( Closed, like every configuration in this package: a key the type does not declare is `unknown_key`, a required one missing is `missing_key`, - both located at the object. An optional key left out is parsed as + both located at the key. An optional key left out is parsed as `UNSET`, so a record never depends on a default for it. """ parsed: dict[str, object] = {} found: list[ValidationProblem] = [] for key in entries: if key not in members: - found.extend(problem(loc, f"unexpected key {key!r}", "unknown_key")) + found.extend(problem((*loc, key), f"unexpected key {key!r}", "unknown_key")) for key, (required, member) in members.items(): if key not in entries: if required: - found.extend(problem(loc, f"missing required key {key!r}", "missing_key")) + found.extend(problem((*loc, key), f"missing required key {key!r}", "missing_key")) else: parsed[key] = UNSET continue @@ -469,7 +473,9 @@ def record_of(record: Callable[..., object], members: Members) -> Parser: """A member that is itself an object with declared keys, built as a dataclass. Built only from an object whose every key read; otherwise the value - comes back as it came, with the reasons. + comes back as it came, with the reasons. A record that refuses its + own values -- a `__post_init__` raising `MetadataValidationError` -- + is reported the same way, located under the object. """ def parse(value: object, loc: Loc) -> Parsed: @@ -479,7 +485,16 @@ def parse(value: object, loc: Loc) -> Parsed: parsed, found = _keys(members, entries, loc) if any(entry.kind != "unknown_key" for entry in found): return entries, found - return record(**parsed), found + try: + return record(**parsed), found + except MetadataValidationError as refused: + return entries, ( + *found, + *( + ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) + for entry in refused.problems + ), + ) return parse @@ -560,7 +575,7 @@ def parser_for(annotation: object, leaf: Leaf = _no_leaf) -> Parser | None: the module docstring. `leaf` is asked first, here and at every depth -- inside a union, an array, an object -- and what it returns is used as it is. Closed: an annotation outside these implies no - parser, and a dataclass declaring one is refused at class creation. + parser, and an entity declaring one is refused at registration. The field is written as one of these shapes instead, with any finer rule in `__post_init__`. """ @@ -585,7 +600,7 @@ def parser_for(annotation: object, leaf: Leaf = _no_leaf) -> Parser | None: # in the process is the one every later one resolves to. The # parse is a membership test either way; this is so the message # listing the values does not depend on import order. - return one_of(tuple(sorted(cast("tuple[str, ...]", get_args(inner))))) + return one_of(tuple(sorted(get_args(inner), key=repr))) if is_union(inner): return _union(inner, leaf) if get_origin(inner) is tuple: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 6a1fca9749..30d66137ac 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -89,6 +89,7 @@ class BytesCodec(ArrayBytesCodec): endian: Endianness | UNSET = UNSET identifier: ClassVar[str] = BYTES_CODEC_NAME + variable_size: ClassVar[bool] = False def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """The data type reaching here must have a raw byte representation. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 873d62f32e..fc7ad47b75 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -17,8 +17,6 @@ ArrayArrayCodec, DataTypeEntity, Opaque, - canonicalized, - written, ) from zarr_metadata.v3._parts import ArrayParts @@ -141,10 +139,11 @@ class CastValueCodec(ArrayArrayCodec): scalar_map: ScalarMap | UNSET = UNSET identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME + variable_size: ClassVar[bool] = False def canonical(self) -> Self: """The target data type in its own canonical form.""" - return replace(self, data_type=canonicalized(self.data_type)) + return replace(self, data_type=self.data_type.canonical()) def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" @@ -152,7 +151,7 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming.with_data_type(data_type if isinstance(data_type, DataTypeEntity) else None) def to_json(self) -> CastValueCodecObject: - configuration: CastValueCodecConfiguration = {"data_type": written(self.data_type)} + configuration: CastValueCodecConfiguration = {"data_type": self.data_type.to_json()} if self.rounding is not UNSET: configuration["rounding"] = self.rounding if self.out_of_range is not UNSET: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 8a57edb6e4..1ed9917444 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -67,6 +67,7 @@ class Crc32cCodec(BytesBytesCodec): """ identifier: ClassVar[str] = CRC32C_CODEC_NAME + variable_size: ClassVar[bool] = False def to_json(self) -> Crc32cCodecName: return "crc32c" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 00126f4c4a..fe313872d9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -102,6 +102,7 @@ class ScaleOffsetCodec(ArrayArrayCodec): scale: JSONValue | UNSET = UNSET identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME + variable_size: ClassVar[bool] = False problems = scale_offset_problems diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 0750b27f2f..257671b331 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -17,9 +17,7 @@ ArrayBytesCodec, CodecEntity, Opaque, - canonicalized, problem, - written, ) from zarr_metadata.v3._parts import ( UNKNOWN_GRID, @@ -130,8 +128,8 @@ def canonical(self) -> Self: """Each pipeline's codecs in their own canonical form.""" return replace( self, - codecs=tuple(canonicalized(codec) for codec in self.codecs), - index_codecs=tuple(canonicalized(codec) for codec in self.index_codecs), + codecs=tuple(codec.canonical() for codec in self.codecs), + index_codecs=tuple(codec.canonical() for codec in self.index_codecs), ) def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: @@ -209,8 +207,8 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation def to_json(self) -> ShardingIndexedCodecObject: configuration: ShardingIndexedCodecConfiguration = { "chunk_shape": self.chunk_shape, - "codecs": tuple(written(codec) for codec in self.codecs), - "index_codecs": tuple(written(codec) for codec in self.index_codecs), + "codecs": tuple(codec.to_json() for codec in self.codecs), + "index_codecs": tuple(codec.to_json() for codec in self.index_codecs), } if self.index_location is not UNSET: configuration["index_location"] = self.index_location diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 4929aa0cd5..86c4feac7e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -85,6 +85,7 @@ class TransposeCodec(ArrayArrayCodec): order: tuple[int, ...] identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME + variable_size: ClassVar[bool] = False problems = transpose_problems diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index b37e39529f..01115045cc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -64,7 +64,7 @@ def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[Validati @dataclass(frozen=True) -class IntegerDataType(DataTypeEntity, base=True): +class IntegerDataType(DataTypeEntity): """A fixed-width integer. The width is the whole difference.""" bounds: ClassVar[tuple[int, int]] @@ -81,7 +81,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class FloatDataType(DataTypeEntity, base=True): +class FloatDataType(DataTypeEntity): """A binary float. A fill value may be a number, a named non-finite, or hex.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -121,7 +121,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class ComplexDataType(DataTypeEntity, base=True): +class ComplexDataType(DataTypeEntity): """A complex number: a `[real, imag]` pair of the component float type.""" scalar_storage: ClassVar[StorageClass] = "multi_byte" @@ -178,7 +178,7 @@ def numpy_time_problems(data_type: NumpyTimeDataType, /) -> Iterator[ValidationP @dataclass(frozen=True) -class NumpyTimeDataType(DataTypeEntity, base=True): +class NumpyTimeDataType(DataTypeEntity): """A numpy time scalar: a signed 64-bit count of units, or `NaT`. The two time types share their configuration -- a unit and a scale diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 1ffc65c337..7219797838 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -18,9 +18,7 @@ Loc, Opaque, StorageClass, - canonicalized, problem, - written, ) if TYPE_CHECKING: @@ -101,7 +99,7 @@ class StructFieldComponent: def _written_field(field: StructFieldComponent) -> StructField: - return {"name": field.name, "data_type": written(field.data_type)} + return {"name": field.name, "data_type": field.data_type.to_json()} def struct_problems(data_type: "StructDataType", /) -> "Iterator[ValidationProblem]": @@ -158,7 +156,7 @@ def canonical(self) -> Self: return replace( self, fields=tuple( - replace(field, data_type=canonicalized(field.data_type)) for field in self.fields + replace(field, data_type=field.data_type.canonical()) for field in self.fields ), ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index d9e665c9ad..bb649c08a2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -42,7 +42,7 @@ `StorageTransformerEntity`; declare the configuration as dataclass fields; write every rule finer than a type as a function of the instance that yields problems, and bind it as `problems`; add the class -to a scope. Complete, and runnable as written: +to a scope. Complete; runnable given a `document`: from collections.abc import Iterator from dataclasses import dataclass @@ -78,6 +78,7 @@ class AcmeLz4Codec(BytesBytesCodec): acceleration: int | UNSET = UNSET # optional: absent reads as UNSET identifier: ClassVar[str] = "acme.lz4" + variable_size: ClassVar[bool] = True # a compressor: its output length is not fixed problems = acme_lz4_problems def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: @@ -117,12 +118,15 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: **What an entity answers for itself**, beyond its fields. `to_json`, abstract: the entity as a document writes it, as a literal of its own TypedDict, which pyright holds to that type -- the bare name when every -member is absent, the object otherwise, a contained entity through -`written`. `canonical`, the entity in its simplest equivalent form: the -entity itself by default, overridden where two spellings of its members -mean the same, and in an entity that contains entities to put those in -canonical form -- `replace(self, inner=canonicalized(self.inner))`. -`coerce` is written once in the base. Then, by kind: +member is absent, the object otherwise, a contained entity through its +own `to_json`. `canonical`, the entity in its simplest equivalent form: +the entity itself by default, overridden where two spellings of its +members mean the same, and in an entity that contains entities to put +those in canonical form -- `replace(self, inner=self.inner.canonical())`. +An `Opaque` answers both as well, with the JSON it kept and with itself, +so a field typed `CodecEntity | Opaque` is written and simplified without +asking which it holds. `coerce` is written once in the base. Then, by +kind: - Every entity: `identifier`, the name it is registered under. A family -- one class for every `acme.fixedN` -- overrides `accepts(name)` and @@ -132,30 +136,31 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: `transition(incoming: ArrayParts) -> ArrayParts | None` -- abstract: return `incoming` if it leaves the array's shape, grid and data type alone, or the parts it hands the next codec -- and any codec may define - `incoming_problems(incoming)` for what it cannot take. `variable_size` - says its output length is not fixed. + `incoming_problems(incoming)` for what it cannot take. Every codec + declares `variable_size`, whether its output length depends on its + input, which is what keeps a compressor out of a shard's index. - A data type: `scalar_storage`, one of `StorageClass` (the `bytes` codec asks it whether an endianness is needed), and `fill_value_problems(value, loc)`, abstract: it judges a document's `fill_value`, and a type that accepts any says so with `return ()`. The families `IntegerDataType`, `FloatDataType`, `ComplexDataType` and `NumpyTimeDataType` carry both for the types they cover; a family of - your own is a subclass declared with `base=True`, which owes nothing - itself and passes its class variables down. + your own is a plain subclass that is never registered itself, and + passes its class variables down. - A chunk grid: `grid(array_shape)`, abstract, and `shape_problems`; see `ChunkGridEntity`. -What a kind leaves abstract, registration refuses an entity for not -defining, as it refuses a class without `@dataclass` and a codec -subclassing `CodecEntity` instead of a kind; class creation refuses a -field whose annotation is not a shape JSON takes -- a nested entity -without `Opaque` among them -- and a class variable a base annotates and -nothing sets. Each says what to write. Everything else an author could -get wrong, pyright says in the editor: the fields, the class variables -and the kind's abstract methods are ordinary typed Python. A scope reads -what a class is off the class: its kind is its base, its key is its -`identifier`, so `extended_with` takes the classes and nothing can be -misfiled. +Registration is the one moment an entity is refused, with a message +that says what to write: a class without `@dataclass`, a codec +subclassing `CodecEntity` instead of a kind, a field whose annotation is +not a shape JSON takes -- a nested entity without `Opaque` among them -- +a `__post_init__` of the entity's own, a class variable a base +annotates and nothing sets, and what a kind leaves abstract. Everything +else an author could get wrong, pyright says in the editor: the fields, +the class variables and the kind's abstract methods are ordinary typed +Python. A scope reads what a class is off the class: its kind is its +base, its key is its `identifier`, so `extended_with` takes the classes +and nothing can be misfiled. **Naming the JSON type.** The return annotation of `to_json` -- above, `AcmeLz4Object | Literal["acme.lz4"]` -- is the entity's own JSON type, @@ -212,12 +217,10 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: Opaque, StorageClass, StorageTransformerEntity, - canonicalized, is_integer, named_configuration, problem, within, - written, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid, Extents from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS, Context @@ -260,11 +263,9 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: "StorageTransformerEntity", "ValidationProblem", "ZarrV3MetadataFieldJSON", - "canonicalized", "chain_problems", "is_integer", "named_configuration", "problem", "within", - "written", ] diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 6b720f36cd..01d75cc10c 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -31,9 +31,7 @@ Opaque, ValidationProblem, ZarrV3MetadataFieldJSON, - canonicalized, problem, - written, ) if TYPE_CHECKING: @@ -72,6 +70,7 @@ class AcmeAffineCodec(ArrayArrayCodec): dtype: DataTypeEntity | Opaque | UNSET = UNSET identifier: ClassVar[str] = "acme.affine" + variable_size: ClassVar[bool] = False problems = acme_affine_problems def canonical(self) -> Self: @@ -79,7 +78,7 @@ def canonical(self) -> Self: return replace( self, offset=UNSET if self.offset == 0 else self.offset, - dtype=UNSET if self.dtype is UNSET else canonicalized(self.dtype), + dtype=UNSET if self.dtype is UNSET else self.dtype.canonical(), ) def to_json(self) -> AcmeAffineObject: @@ -87,7 +86,7 @@ def to_json(self) -> AcmeAffineObject: if self.offset is not UNSET: configuration["offset"] = self.offset if self.dtype is not UNSET: - configuration["dtype"] = written(self.dtype) + configuration["dtype"] = self.dtype.to_json() return {"name": "acme.affine", "configuration": configuration} def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 5bcac47b70..d16156e1a3 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -40,9 +40,7 @@ StorageClass, ValidationProblem, ZarrV3MetadataFieldJSON, - canonicalized, problem, - written, ) if TYPE_CHECKING: @@ -162,12 +160,17 @@ def test_a_registered_entity_canonicalizes_itself() -> None: def test_error_an_entity_must_say_what_it_is() -> None: + @dataclass(frozen=True) + class Nameless(BytesBytesCodec): + """A codec that forgot to say what it is.""" + + variable_size: ClassVar[bool] = True + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.nameless" + with pytest.raises(TypeError, match="does not declare identifier"): - # Never bound: the guard raises while the class is being created, - # which is the whole point -- so pyright cannot see it used. - @dataclass(frozen=True) - class Nameless(BytesBytesCodec): - """A codec that forgot to say what it is.""" + CORE_AND_EXTENSIONS.extended_with(Nameless) def test_the_entity_layer_answers_what_a_reader_needs() -> None: @@ -196,6 +199,8 @@ class Defaulted(BytesBytesCodec): identifier: ClassVar[str] = "acme.defaulted" + variable_size: ClassVar[bool] = False + def to_json(self) -> ZarrV3MetadataFieldJSON: if self.level is UNSET: return "acme.defaulted" @@ -275,14 +280,15 @@ def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: # `bounds` is annotated on `IntegerDataType` and bound by none of it, # so every concrete integer type owes one. Nothing lists it: the # requirement is read off the annotation. - with pytest.raises(TypeError, match="does not declare bounds"): + @dataclass(frozen=True) + class Int24DataType(IntegerDataType): + identifier: ClassVar[str] = "acme.int24" - @dataclass(frozen=True) - class Int24DataType(IntegerDataType): - identifier: ClassVar[str] = "acme.int24" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.int24" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.int24" + with pytest.raises(TypeError, match="does not declare bounds"): + CORE_AND_EXTENSIONS.extended_with(Int24DataType) # A third-party *family*: one class covering a parameterized set of names, @@ -290,6 +296,14 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: ACME_FIXED_PATTERN = re.compile(r"acme\.fixed(\d+)") +def acme_fixed_problems(data_type: AcmeFixedDataType, /) -> Iterator[ValidationProblem]: + match = ACME_FIXED_PATTERN.fullmatch(data_type.data_type_name) + if match is not None and int(match.group(1)) % 8 != 0: + yield ValidationProblem( + ("data_type_name",), "expected a width that is a multiple of 8", "invalid_value" + ) + + @dataclass(frozen=True) class AcmeFixedDataType(DataTypeEntity): """`acme.fixedN`, a fixed-width type for every N.""" @@ -298,6 +312,7 @@ class AcmeFixedDataType(DataTypeEntity): identifier: ClassVar[str] = "acme.fixed" scalar_storage: ClassVar[StorageClass] = "multi_byte" + problems = acme_fixed_problems @classmethod def accepts(cls, name: str) -> bool: @@ -326,19 +341,30 @@ def test_a_third_party_can_register_a_family() -> None: # near-miss is still nobody's. assert scope.resolve(DataTypeEntity, AcmeFixedDataType.identifier) is None assert scope.resolve(DataTypeEntity, "acme.fixed") is None + # A rule about the name lands on the field: the document has no + # configuration to locate it under. + problems = validate_array_metadata_v3( + _document(data_type="acme.fixed12", fill_value=0), context=scope + ) + assert [(p.loc, p.kind) for p in problems] == [(("data_type",), "invalid_value")] def test_error_a_member_needs_a_check_from_somewhere() -> None: # An annotation outside the shapes the parser reads implies no # parser, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. - with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): + @dataclass(frozen=True) + class Structured(BytesBytesCodec): + inner: object - @dataclass(frozen=True) - class Structured(BytesBytesCodec): - inner: object + identifier: ClassVar[str] = "acme.structured" + variable_size: ClassVar[bool] = False + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.structured" - identifier: ClassVar[str] = "acme.structured" + with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): + CORE_AND_EXTENSIONS.extended_with(Structured) # A third-party codec that contains another codec. @@ -350,11 +376,13 @@ class AcmeWrapperCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.wrapper" + variable_size: ClassVar[bool] = False + def canonical(self) -> Self: - return replace(self, inner=canonicalized(self.inner)) + return replace(self, inner=self.inner.canonical()) def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.wrapper", "configuration": {"inner": written(self.inner)}} + return {"name": "acme.wrapper", "configuration": {"inner": self.inner.to_json()}} def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: @@ -427,15 +455,17 @@ class AcmeFramedCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.framed" + variable_size: ClassVar[bool] = False + def canonical(self) -> Self: return replace( self, - inner=canonicalized(self.inner), + inner=self.inner.canonical(), frame=UNSET if self.frame == 0 else self.frame, ) def to_json(self) -> ZarrV3MetadataFieldJSON: - configuration: dict[str, JSONValue] = {"inner": written(self.inner)} + configuration: dict[str, JSONValue] = {"inner": self.inner.to_json()} if self.frame is not UNSET: configuration["frame"] = self.frame return {"name": "acme.framed", "configuration": configuration} @@ -449,13 +479,18 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: def test_error_a_nested_field_names_a_kind() -> None: # `MetadataEntity` is of no kind, so a field typed as one could not # be resolved through any scope. - with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): + @dataclass(frozen=True) + class Vague(BytesBytesCodec): + inner: MetadataEntity | Opaque + + identifier: ClassVar[str] = "acme.vague" + variable_size: ClassVar[bool] = False - @dataclass(frozen=True) - class Vague(BytesBytesCodec): - inner: MetadataEntity | Opaque + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.vague" - identifier: ClassVar[str] = "acme.vague" + with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): + CORE_AND_EXTENSIONS.extended_with(Vague) # The JSON types third-party entities name, at module level so their @@ -495,6 +530,8 @@ class AcmeBlockCodec(BytesBytesCodec): block: int identifier: ClassVar[str] = "acme.block" + + variable_size: ClassVar[bool] = False problems = acme_block_problems def to_json(self) -> ZarrV3MetadataFieldJSON: @@ -543,6 +580,8 @@ class AcmeRangeCodec(BytesBytesCodec): high: int identifier: ClassVar[str] = "acme.range" + + variable_size: ClassVar[bool] = False problems = acme_range_problems def to_json(self) -> ZarrV3MetadataFieldJSON: @@ -567,25 +606,83 @@ def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one def test_error_an_entity_may_not_define_post_init() -> None: # `coerce` never runs it, so a rule written there would judge a # hand-built entity and no document. + @dataclass(frozen=True) + class Checked(BytesBytesCodec): + identifier: ClassVar[str] = "acme.checked" + variable_size: ClassVar[bool] = False + + def __post_init__(self) -> None: + return None + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.checked" + with pytest.raises(TypeError, match="defines __post_init__; write its rules as a function"): + CORE_AND_EXTENSIONS.extended_with(Checked) + + +def test_error_a_field_annotation_names_what_is_not_defined() -> None: + # Annotations are resolved where the class is, at registration; a + # type defined inside a function is not there. + class Local(TypedDict, closed=True): + depth: int + + @dataclass(frozen=True) + class Localized(BytesBytesCodec): + inner: Local - @dataclass(frozen=True) - class Checked(BytesBytesCodec): - identifier: ClassVar[str] = "acme.checked" + identifier: ClassVar[str] = "acme.localized" + variable_size: ClassVar[bool] = False - def __post_init__(self) -> None: - return None + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.localized" + + with pytest.raises(TypeError, match="a field annotation names 'Local', which is not defined"): + CORE_AND_EXTENSIONS.extended_with(Localized) + + +def test_error_a_codec_says_whether_its_output_size_is_fixed() -> None: + # A default in either direction is a verdict: a compressor that + # said nothing would be accepted as a shard-index codec. + @dataclass(frozen=True) + class Sizeless(BytesBytesCodec): + identifier: ClassVar[str] = "acme.sizeless" + + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.sizeless" + + with pytest.raises(TypeError, match="does not declare variable_size"): + CORE_AND_EXTENSIONS.extended_with(Sizeless) + + +def test_a_reader_gets_structural_and_semantic_reasons_together() -> None: + with pytest.raises(MetadataValidationError) as caught: + ArrayDocumentV3.from_json(_document(attributes=5, fill_value=-1)) + assert {problem.loc for problem in caught.value.problems} == {("attributes",), ("fill_value",)} + + +def test_a_malformed_envelope_is_one_problem() -> None: + # The envelope is judged once, by the scope; the entity is not asked + # to read what is not a metadata field. + _, problems = CORE_AND_EXTENSIONS.coerce(CodecEntity, 5, ("codecs", 0)) + assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0), "invalid_type")] + _, problems = CORE_AND_EXTENSIONS.coerce( + CodecEntity, {"name": "gzip", "configuration": 42}, ("codecs", 0) + ) + assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "invalid_type")] def test_a_slotted_entity_is_accepted() -> None: - # `@dataclass(slots=True)` builds the class twice, so class creation - # sees it twice; the second time its members are slot descriptors. + # `@dataclass(slots=True)` builds the class twice; registration sees + # the second, whose members are slot descriptors. @dataclass(frozen=True, slots=True) class AcmeSlotted(BytesBytesCodec): level: int identifier: ClassVar[str] = "acme.slotted" + variable_size: ClassVar[bool] = False + def to_json(self) -> ZarrV3MetadataFieldJSON: return {"name": "acme.slotted", "configuration": {"level": self.level}} @@ -599,6 +696,7 @@ def test_a_bare_class_var_is_a_class_variable() -> None: @dataclass(frozen=True) class AcmeNoted(BytesBytesCodec): identifier: ClassVar[str] = "acme.noted" + variable_size: ClassVar[bool] = False note: ClassVar = "not a member" def to_json(self) -> ZarrV3MetadataFieldJSON: @@ -617,6 +715,8 @@ class AcmeScaled(ArrayArrayCodec): identifier: ClassVar[str] = "acme.scaled" + variable_size: ClassVar[bool] = False + def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming @@ -656,13 +756,18 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: def test_error_a_nested_field_admits_opaque() -> None: # What the field holds when the inner name is out of scope. - with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): + @dataclass(frozen=True) + class Closed(BytesBytesCodec): + inner: CodecEntity - @dataclass(frozen=True) - class Closed(BytesBytesCodec): - inner: CodecEntity + identifier: ClassVar[str] = "acme.closed" + variable_size: ClassVar[bool] = False - identifier: ClassVar[str] = "acme.closed" + def to_json(self) -> ZarrV3MetadataFieldJSON: + return "acme.closed" + + with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): + CORE_AND_EXTENSIONS.extended_with(Closed) def test_error_an_array_array_codec_defines_transition() -> None: @@ -671,6 +776,7 @@ def test_error_an_array_array_codec_defines_transition() -> None: @dataclass(frozen=True) class Silent(ArrayArrayCodec): identifier: ClassVar[str] = "acme.silent" + variable_size: ClassVar[bool] = False def to_json(self) -> ZarrV3MetadataFieldJSON: return "acme.silent" From 01cb435d7def14f7caf24dfad8eea8d66373156f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 17:17:31 +0200 Subject: [PATCH 087/107] perf(zarr-metadata): a parser is compiled once per class; the reading is an argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A parser took the scope it read nested entities in through a closure, so nothing compiled could outlive one read: every read resolved the class's annotations and compiled every field's parser again, and a document read cost eighty times a `json.loads` of the same bytes. The parser now takes the reading it runs in as an argument, generic in what the caller passes -- `Parser[S]` is `(value, loc, state) -> parsed` -- and hands it down untouched to the parsers it is built from. The entity layer's one shape, a field holding another entity, reads its scope and its problem sink off a `_Reading`. So a class's plan -- each field's parser, whether it is optional, whether the envelope's name fills it -- is a pure function of the class, compiled once and cached with `functools.cache`, as the class's resolved annotations now are. The scope's `coerce` also stops re-judging an envelope the model layer has judged. Measured on a document with a shard, six codecs and a nested pipeline: 199 µs to 85 µs per read; one codec, 21 µs to 6 µs. The 40,000-document differential is unchanged to the problem. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 9 ++ .../src/zarr_metadata/v3/_entity.py | 116 +++++++++++------ .../src/zarr_metadata/v3/_registry.py | 17 ++- .../src/zarr_metadata/v3/_typed_json.py | 120 ++++++++++-------- 4 files changed, 163 insertions(+), 99 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index dfdad2673b..fc1709f9cc 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -133,3 +133,12 @@ fails on a comparison, a fixed-length array accepts a list as a homogeneous one does, a key inside a nested object is reported at the key like a top-level one, and a record dataclass that refuses its own values is reported rather than raised. + +A parser is compiled once per class and run many times. Each takes the +reading it runs in as an argument -- for the one shape the entity layer +adds, a field holding another entity, that is the scope to read it in +and where its problems go -- so nothing that varies between reads is +compiled into it, and the plan for a class, like its resolved field +annotations, is a pure function of the class and cached as one. A +document with a shard, six codecs and a nested pipeline reads in 85 µs +where it took 199, and one codec in 6 µs where it took 21. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 7cc2fce406..de1f76df9a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -27,6 +27,7 @@ from __future__ import annotations +import functools from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass @@ -57,7 +58,6 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._parts import ArrayParts, ChunkGrid from zarr_metadata.v3._registry import Context - from zarr_metadata.v3._typed_json import Leaf EntityT = TypeVar("EntityT", bound="MetadataEntity") @@ -226,7 +226,7 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: unread: list[str] = [] for name, annotation in hints.items(): try: - accepted = parser_for(annotation, _reading(None, [])) is not None + accepted = parser_for(annotation, _nested_field) is not None except TypeError as refused: return f"{cls.__name__}: {name} {refused}" if not accepted: @@ -283,34 +283,72 @@ def nested_kind(annotation: object) -> type[MetadataEntity] | None: raise TypeError(msg) -def _reading(context: Context | None, nested: list[ValidationProblem]) -> Leaf: - """How a field holding another entity is parsed: `Kind | Opaque`, read in `context`. +@dataclass(frozen=True, slots=True) +class _Reading: + """What one reading hands down into the fields that hold entities. - Asked by the parser at every depth, so a struct's fields' data types - and a shard's inner pipelines are read the same way. The envelope's - shape is the containing field's own problem; what is found inside the - entity is collected in `nested`, apart, because the containing - entity's rules still run over its own members when only a contained - entity is wrong. With no `context` the shape is checked and nothing - is read, which is what registration asks. + The scope the inner entities are read in, and the problems found + inside them, kept apart from the containing entity's own: its rules + still run over its own members when only a contained entity is + wrong. """ - def leaf(annotation: object) -> Parser | None: - kind = nested_kind(annotation) - if kind is None: - return None + context: Context + nested: list[ValidationProblem] + + +def _nested_field(annotation: object) -> Parser[_Reading] | None: + """The parser for a field holding another entity: `Kind | Opaque`, read in the reading's scope. - def parse(value: object, loc: Loc) -> Parsed: - problems = is_metadata_field(value, loc) - if len(problems) != 0 or context is None: - return value, problems - entity, found = context.coerce(kind, value, loc) - nested.extend(found) - return entity, () + The entity layer's one shape of its own, asked by the parser at + every depth, so a struct's fields' data types and a shard's inner + pipelines are read the same way. The envelope's shape is the + containing field's own problem; what is found inside the entity + goes to the reading. + """ + kind = nested_kind(annotation) + if kind is None: + return None - return parse + def parse(value: object, loc: Loc, reading: _Reading) -> Parsed: + problems = is_metadata_field(value, loc) + if len(problems) != 0: + return value, problems + entity, found = reading.context.coerce(kind, value, loc) + reading.nested.extend(found) + return entity, () + + return parse + + +@dataclass(frozen=True, slots=True) +class _Member: + """How `coerce` reads one field: what the annotation says, compiled once.""" - return leaf + key: str + from_name: bool + optional: bool + parse: Parser[_Reading] + + +@functools.cache +def _plan(cls: type[MetadataEntity]) -> tuple[_Member, ...]: + """The fields of `cls` as `coerce` reads them, compiled once per class. + + A pure function of the class: its fields are fixed once it exists, + and each parser is a function of its annotation alone, taking the + reading it runs in as an argument. `TypeError` for a field no parser + reads, which registration refuses first. + """ + return tuple( + _Member( + key, + is_from_name(annotation), + is_optional(annotation), + parser(annotation, _nested_field), + ) + for key, annotation in field_hints(cls).items() + ) @dataclass(frozen=True) @@ -406,31 +444,29 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: return None, problem((), f"expected the {cls.identifier!r} entity") if len(envelope) != 0: return None, envelope - hints = field_hints(cls) - if given is None and any( - not is_from_name(annotation) and not is_optional(annotation) - for annotation in hints.values() - ): + plan = _plan(cls) + if given is None and any(not member.from_name and not member.optional for member in plan): return None, problem( ("configuration",), f"{cls.identifier!r} requires a configuration", "missing_key", ) configuration: Mapping[str, object] = {} if given is None else given - nested: list[ValidationProblem] = [] - reading = _reading(context, nested) + reading = _Reading(context, []) + declared = {member.key: member for member in plan} members: dict[str, object] = {} own: list[ValidationProblem] = [] for key in configuration: - if key not in hints or is_from_name(hints[key]): + if key not in declared or declared[key].from_name: own.extend( problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") ) - for key, annotation in hints.items(): - if is_from_name(annotation): + for member in plan: + key = member.key + if member.from_name: members[key] = name elif key not in configuration: - if is_optional(annotation): + if member.optional: members[key] = UNSET else: own.extend( @@ -442,11 +478,11 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # Arrays as tuples before parsing, so a member holds the # tuples its type declares, never the lists raw JSON # arrives as. - members[key], problems = parser(annotation, reading)( - as_tuples(configuration[key]), ("configuration", key) + members[key], problems = member.parse( + as_tuples(configuration[key]), ("configuration", key), reading ) own.extend(problems) - found = (*own, *nested) + found = (*own, *reading.nested) if any(entry.kind != "unknown_key" for entry in own): # An unknown key is survivable; a member that could not be # read is a hole, and judging around it would be guessing. @@ -455,7 +491,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # A problem about a member the envelope's name carries is about # the entity, and lands on it rather than under a configuration # the document does not have. - from_name = {key for key, annotation in hints.items() if is_from_name(annotation)} + from_name = {member.key for member in plan if member.from_name} refused = within( (), tuple( @@ -469,7 +505,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # Values the spec disallows: reported rather than raised, # every one, located under the configuration. return None, (*found, *refused) - if any(entry.kind != "unknown_key" for entry in nested): + if any(entry.kind != "unknown_key" for entry in reading.nested): # A contained entity could not be read. This entity's own # rules ran -- an invalid inner is an `Opaque`, as an # out-of-scope one is -- but what is handed back is not an diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 5e95b96af6..7baaaa2c02 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -173,21 +173,20 @@ def coerce( envelope gets the same structural judgment here that the model layer gives a top-level one -- an extra member, a `configuration` that is not an object, a `must_understand` that is not a boolean - or is `false`. `envelope_judged` says the model layer has - reported that already, which it has for the fields of a - document, so it is not reported twice. The entity is read - whenever there is one to read -- a stray member or a malformed - `must_understand` says nothing about the configuration -- and - not when the value names no entity or its configuration is not - an object, which the envelope judgment has already said. + or is `false`. `envelope_judged` says the model layer has judged + and reported that already, which it has for the fields of a + document, so it is neither judged nor reported twice. The entity + is read whenever there is one to read -- a stray member or a + malformed `must_understand` says nothing about the configuration + -- and not when the value names no entity or its configuration + is not an object, which the envelope judgment has said. """ - envelope = validate_metadata_field_v3(value, allow_must_understand_false=False) problems = ( () if envelope_judged else tuple( ValidationProblem((*loc, *found.loc), found.message, found.kind) - for found in envelope + for found in validate_metadata_field_v3(value, allow_must_understand_false=False) ) ) name, _, malformed = named_configuration(value) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index 3b71a9eb8f..9d4e25aacc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -15,11 +15,14 @@ Nothing here knows what a metadata entity is. A caller with a shape of its own -- a field that holds another entity, read through a scope -- passes a `leaf`, which is asked first for every annotation at every -depth; the parser it returns is used as it is. +depth; the parser it returns is used as it is. A parser is compiled +once per annotation and takes the reading it runs in as an argument, +so what varies between reads never has to be compiled into it. """ from __future__ import annotations +import functools import sys import types from collections.abc import Callable, Mapping, Sequence @@ -33,6 +36,7 @@ NotRequired, Required, TypeAlias, + TypeVar, Union, cast, get_args, @@ -50,16 +54,25 @@ from zarr_metadata.model._validation import ProblemKind -Loc: TypeAlias = "tuple[str | int, ...]" +S = TypeVar("S") +"""The reading a parser runs in: whatever the caller hands through, untouched here.""" + +Loc: TypeAlias = tuple[str | int, ...] """Where in a document a value sits: the keys and indices down to it.""" -Parsed: TypeAlias = "tuple[object, tuple[ValidationProblem, ...]]" +Parsed: TypeAlias = tuple[object, tuple[ValidationProblem, ...]] """What a parser returns: the typed value, and every problem found with it.""" -Parser: TypeAlias = "Callable[[object, Loc], Parsed]" -"""One value against one annotation, located at `loc`.""" +Parser: TypeAlias = Callable[[object, Loc, S], Parsed] +"""One value against one annotation, located at `loc`, in a reading `S`. + +A parser is compiled once from the annotation and run many times; what +varies between runs -- for a caller whose leaf reads a nested entity, +the scope to read it in -- is the reading, an argument every parser +hands down to the parsers it is built from and reads nothing of itself. +""" -Leaf: TypeAlias = "Callable[[object], Parser | None]" +Leaf: TypeAlias = Callable[[object], "Parser[S] | None"] """A caller's own shapes: asked first for every annotation, None to decline.""" @@ -195,6 +208,7 @@ def is_class_var(annotation: object) -> bool: return annotation is ClassVar or get_origin(annotation) is ClassVar +@functools.cache def field_hints(cls: type) -> dict[str, object]: """The dataclass fields of `cls`, resolved, base first. @@ -203,6 +217,10 @@ def field_hints(cls: type) -> dict[str, object]: `ClassVar` whose annotation names something imported only for the type checker cannot fail registration. `@dataclass` sees the same set, in the same order. + + Cached per class: a class's annotations are fixed once it exists, + and resolving them costs a third of a read. A name that does not + resolve raises, and a raise is not cached. """ hints: dict[str, object] = {} for ancestor in reversed(cls.__mro__): @@ -319,8 +337,8 @@ def has_shape(shape: str | None, value: object) -> bool: # --- the parsers --------------------------------------------------------- -def _scalar(description: str, admits: Callable[[object], bool]) -> Parser: - def parse(value: object, loc: Loc) -> Parsed: +def _scalar(description: str, admits: Callable[[object], bool]) -> Parser[object]: + def parse(value: object, loc: Loc, state: object) -> Parsed: if admits(value): return value, () return value, problem(loc, f"expected {description}, got {value!r}") @@ -328,23 +346,23 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -_INTEGER: Parser = _scalar("an integer", is_integer) -_NUMBER: Parser = _scalar( +_INTEGER: Parser[object] = _scalar("an integer", is_integer) +_NUMBER: Parser[object] = _scalar( "a number", lambda value: not isinstance(value, bool) and isinstance(value, (int, float)) ) -_BOOLEAN: Parser = _scalar("a boolean", lambda value: isinstance(value, bool)) -_STRING: Parser = _scalar("a string", lambda value: isinstance(value, str)) -_JSON: Parser = _scalar("a JSON value", is_json) +_BOOLEAN: Parser[object] = _scalar("a boolean", lambda value: isinstance(value, bool)) +_STRING: Parser[object] = _scalar("a string", lambda value: isinstance(value, str)) +_JSON: Parser[object] = _scalar("a JSON value", is_json) -def one_of(allowed: tuple[object, ...]) -> Parser: +def one_of(allowed: tuple[object, ...]) -> Parser[object]: """A member whose type is a closed set of values. Equal and of the same type: JSON `true` is not the integer 1, though Python says `True == 1`. """ - def parse(value: object, loc: Loc) -> Parsed: + def parse(value: object, loc: Loc, state: object) -> Parsed: if not any(value == entry and type(value) is type(entry) for entry in allowed): return value, problem( loc, f"expected one of {allowed!r}, got {value!r}", "invalid_value" @@ -354,17 +372,17 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -def sequence_of(element: Parser) -> Parser: +def sequence_of(element: Parser[S]) -> Parser[S]: """A member whose type is an array of one element type, parsed element by element.""" - def parse(value: object, loc: Loc) -> Parsed: + def parse(value: object, loc: Loc, state: S) -> Parsed: if not isinstance(value, (list, tuple)): return value, problem(loc, f"expected a sequence, got {value!r}") entries = cast("list[object] | tuple[object, ...]", value) parsed: list[object] = [] found: list[ValidationProblem] = [] for index, entry in enumerate(entries): - item, problems = element(entry, (*loc, index)) + item, problems = element(entry, (*loc, index), state) parsed.append(item) found.extend(problems) return tuple(parsed), tuple(found) @@ -372,10 +390,10 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -def fixed_tuple(elements: Sequence[Parser], description: str) -> Parser: +def fixed_tuple(elements: Sequence[Parser[S]], description: str) -> Parser[S]: """A member whose type is an array of a fixed length, parsed position by position.""" - def parse(value: object, loc: Loc) -> Parsed: + def parse(value: object, loc: Loc, state: S) -> Parsed: if not isinstance(value, (list, tuple)): return value, problem(loc, f"expected {description}, got {value!r}") entries = tuple(cast("list[object] | tuple[object, ...]", value)) @@ -384,7 +402,7 @@ def parse(value: object, loc: Loc) -> Parsed: parsed: list[object] = [] found: list[ValidationProblem] = [] for position, (element, entry) in enumerate(zip(elements, entries, strict=True)): - item, problems = element(entry, (*loc, position)) + item, problems = element(entry, (*loc, position), state) parsed.append(item) found.extend(problems) return tuple(parsed), tuple(found) @@ -392,7 +410,7 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -def any_of(branches: Sequence[tuple[object, Parser]], description: str) -> Parser: +def any_of(branches: Sequence[tuple[object, Parser[S]]], description: str) -> Parser[S]: """A member whose type is a union of shapes, parsed by the branch it fits. The branch whose top-level shape the value has is the one that @@ -403,12 +421,12 @@ def any_of(branches: Sequence[tuple[object, Parser]], description: str) -> Parse reported by the first that does not. """ - def parse(value: object, loc: Loc) -> Parsed: + def parse(value: object, loc: Loc, state: S) -> Parsed: first: Parsed | None = None for annotation, branch in branches: if not has_shape(shape_of(annotation), value): continue - result = branch(value, loc) + result = branch(value, loc, state) if len(result[1]) == 0: return result if first is None: @@ -420,12 +438,12 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -Members: TypeAlias = "Mapping[str, tuple[bool, Parser]]" +Members: TypeAlias = Mapping[str, tuple[bool, "Parser[S]"]] """An object's declared keys: whether each is required, and its parser.""" def _keys( - members: Members, entries: Mapping[str, object], loc: Loc + members: Members[S], entries: Mapping[str, object], loc: Loc, state: S ) -> tuple[dict[str, object], tuple[ValidationProblem, ...]]: """The declared keys of one object, each parsed at its own key. @@ -446,30 +464,30 @@ def _keys( else: parsed[key] = UNSET continue - item, problems = member(entries[key], (*loc, key)) + item, problems = member(entries[key], (*loc, key), state) parsed[key] = item found.extend(problems) return parsed, tuple(found) -def object_of(members: Members) -> Parser: +def object_of(members: Members[S]) -> Parser[S]: """A member that is itself an object with declared keys, kept as the mapping it came as. A key the type does not declare is reported and kept: the member still says what the document said. """ - def parse(value: object, loc: Loc) -> Parsed: + def parse(value: object, loc: Loc, state: S) -> Parsed: if not isinstance(value, Mapping): return value, problem(loc, f"expected an object, got {value!r}") entries = cast("Mapping[str, object]", value) - parsed, found = _keys(members, entries, loc) + parsed, found = _keys(members, entries, loc, state) return {**entries, **{key: item for key, item in parsed.items() if key in entries}}, found return parse -def record_of(record: Callable[..., object], members: Members) -> Parser: +def record_of(record: Callable[..., object], members: Members[S]) -> Parser[S]: """A member that is itself an object with declared keys, built as a dataclass. Built only from an object whose every key read; otherwise the value @@ -478,11 +496,11 @@ def record_of(record: Callable[..., object], members: Members) -> Parser: is reported the same way, located under the object. """ - def parse(value: object, loc: Loc) -> Parsed: + def parse(value: object, loc: Loc, state: S) -> Parsed: if not isinstance(value, Mapping): return value, problem(loc, f"expected an object, got {value!r}") entries = cast("Mapping[str, object]", value) - parsed, found = _keys(members, entries, loc) + parsed, found = _keys(members, entries, loc, state) if any(entry.kind != "unknown_key" for entry in found): return entries, found try: @@ -499,21 +517,21 @@ def parse(value: object, loc: Loc) -> Parsed: return parse -def mapping_of(value: Parser) -> Parser: +def mapping_of(value: Parser[S]) -> Parser[S]: """A member whose type is an object with any keys, parsed value by value. The open counterpart of `object_of`: a `Mapping[str, V]` says nothing about which keys there are, only what each value must be. """ - def parse(candidate: object, loc: Loc) -> Parsed: + def parse(candidate: object, loc: Loc, state: S) -> Parsed: if not isinstance(candidate, Mapping): return candidate, problem(loc, f"expected an object, got {candidate!r}") entries = cast("Mapping[str, object]", candidate) parsed: dict[str, object] = {} found: list[ValidationProblem] = [] for key, entry in entries.items(): - item, problems = value(entry, (*loc, key)) + item, problems = value(entry, (*loc, key), state) parsed[key] = item found.extend(problems) return parsed, tuple(found) @@ -524,27 +542,27 @@ def parse(candidate: object, loc: Loc) -> Parsed: # --- the compiler -------------------------------------------------------- -def _members_of(annotations: Mapping[str, object], leaf: Leaf) -> Members | None: +def _members_of(annotations: Mapping[str, object], leaf: Leaf[S]) -> Members[S] | None: """A member table for an object's keys; None if any key's type has no parser.""" - members: dict[str, tuple[bool, Parser]] = {} + members: dict[str, tuple[bool, Parser[S]]] = {} for key, annotation in annotations.items(): - parser = parser_for(annotation, leaf) - if parser is None: + member = parser_for(annotation, leaf) + if member is None: return None required = not is_not_required(annotation) and not is_optional(annotation) - members[key] = (required, parser) + members[key] = (required, member) return members -def _union(inner: object, leaf: Leaf) -> Parser | None: +def _union(inner: object, leaf: Leaf[S]) -> Parser[S] | None: compiled = [(branch, parser_for(branch, leaf)) for branch in get_args(inner)] - branches = [(branch, parser) for branch, parser in compiled if parser is not None] + branches = [(branch, member) for branch, member in compiled if member is not None] if len(branches) != len(compiled): return None return any_of(branches, describe(inner)) -def _tuple(inner: object, leaf: Leaf) -> Parser | None: +def _tuple(inner: object, leaf: Leaf[S]) -> Parser[S] | None: arguments = get_args(inner) if len(arguments) == 2 and arguments[1] is Ellipsis: element = parser_for(arguments[0], leaf) @@ -556,7 +574,7 @@ def _tuple(inner: object, leaf: Leaf) -> Parser | None: return fixed_tuple(elements, describe(inner)) -def _mapping(inner: object, leaf: Leaf) -> Parser | None: +def _mapping(inner: object, leaf: Leaf[S]) -> Parser[S] | None: arguments = get_args(inner) if len(arguments) != 2 or arguments[0] is not str: return None @@ -564,19 +582,20 @@ def _mapping(inner: object, leaf: Leaf) -> Parser | None: return None if value is None else mapping_of(value) -def _no_leaf(annotation: object) -> Parser | None: +def no_leaf(annotation: object) -> Parser[object] | None: + """The leaf of a caller with no shapes of its own.""" return None -def parser_for(annotation: object, leaf: Leaf = _no_leaf) -> Parser | None: +def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: """The parser a field annotation implies, or None if it implies none. A small compiler over the shapes JSON takes and no others, listed in the module docstring. `leaf` is asked first, here and at every depth -- inside a union, an array, an object -- and what it returns is used as it is. Closed: an annotation outside these implies no - parser, and an entity declaring one is refused at registration. - The field is written as one of these shapes instead, with any finer + parser, and an entity declaring one is refused at registration. The + field is written as one of these shapes instead, with any finer rule in `__post_init__`. """ inner = without_unset(strip_annotation(annotation)[0]) @@ -622,7 +641,7 @@ def parser_for(annotation: object, leaf: Leaf = _no_leaf) -> Parser | None: return None -def parser(annotation: object, leaf: Leaf = _no_leaf) -> Parser: +def parser(annotation: object, leaf: Leaf[S]) -> Parser[S]: """The parser a field annotation implies; `TypeError` if it implies none.""" found = parser_for(annotation, leaf) if found is None: @@ -650,6 +669,7 @@ def parser(annotation: object, leaf: Leaf = _no_leaf) -> Parser: "is_optional", "is_union", "mapping_of", + "no_leaf", "object_of", "one_of", "own_annotations", From 9725a4f1f92e06a23f6185cb71395f3f75526992 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 17:37:10 +0200 Subject: [PATCH 088/107] refactor(zarr-metadata): the unchecked record is a public `create_unchecked` The constructor is the checked way to build an entity and stops at the first problem; `coerce` builds the record without the check and asks `problems` for every one. That second way was a private `_unchecked` taking a mapping. It is now `create_unchecked(**members)`, a documented classmethod, for any reader that judges afterwards and wants every problem of a hand-built entity -- the pair pydantic spells `__init__` and `model_construct`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.10.md | 3 ++- .../src/zarr_metadata/v3/_entity.py | 16 ++++++++++------ .../zarr-metadata/src/zarr_metadata/v3/entity.py | 2 ++ .../zarr-metadata/tests/v3/test_extension_api.py | 4 ++++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index 86e9a4db12..78019a05fc 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -10,7 +10,8 @@ disallows as it finds them, bound on the class as `problems`. The constructor stops at the first and raises `MetadataValidationError`; `coerce` runs it to the end and reports every problem in the document instead of raising; a consumer holding an entity may run it too, and -stop or collect as it likes. +stop or collect as it likes. `create_unchecked(**members)` builds the +record without the check, for a reader that judges afterwards. `must_understand` moves to a class variable and out of the configuration entirely. It is a property of the *kind* of metadata -- a codec is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index de1f76df9a..acb35f72c4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -406,11 +406,15 @@ def __post_init__(self) -> None: raise MetadataValidationError((first,)) @classmethod - def _unchecked(cls, members: Mapping[str, object]) -> Self: - """The instance `cls(**members)` would build, without asking `problems`. - - For `coerce`, which asks `problems` itself and reports every one, - where the constructor stops at the first. + def create_unchecked(cls, **members: object) -> Self: + """The record `cls(**members)` would build, without asking `problems`. + + The constructor is the checked way to build an entity, and stops + at the first problem; this is for a reader that judges + afterwards and wants every one, as `coerce` does -- it asks + `problems` itself and reports what it yields. The members are + the caller's promise: nothing here checks their names or types, + which the constructor does. """ entity = object.__new__(cls) for name, value in members.items(): @@ -487,7 +491,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # An unknown key is survivable; a member that could not be # read is a hole, and judging around it would be guessing. return None, found - entity = cls._unchecked(members) + entity = cls.create_unchecked(**members) # A problem about a member the envelope's name carries is about # the entity, and lands on it rather than under a configuration # the document does not have. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index bb649c08a2..12913f208e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -112,6 +112,8 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: constructor stops at the first problem it yields, so `AcmeLz4Codec(acceleration=0)` raises `MetadataValidationError`; `coerce` runs it to the end and reports every problem in the document. +`create_unchecked(**members)` builds the record without the check, for +a reader that judges afterwards with `problems` and wants every one. It runs only on an entity whose members all read: a member of the wrong type is reported and the entity is not built. diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index d16156e1a3..eaee235d59 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -601,6 +601,10 @@ def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one ) assert [p.loc for p in problems] == [("configuration", "low"), ("configuration", "high")] assert list(acme_range_problems(AcmeRangeCodec(low=0, high=1))) == [] + # A reader that wants every problem of a hand-built one builds the + # record without the check and asks. + record = AcmeRangeCodec.create_unchecked(low=-1, high=-2) + assert [p.loc for p in record.problems()] == [("low",), ("high",)] def test_error_an_entity_may_not_define_post_init() -> None: From e4db40fde48205e31f0246cac970e0645a60f927 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 18:09:21 +0200 Subject: [PATCH 089/107] refactor(zarr-metadata): to_json is written once, from the fields `writer_for` is the parser's inverse over the same field annotation: what `coerce` reads from a document, `to_json` puts back. The base writes every entity -- the bare name when every member it holds is absent, the object otherwise, a contained entity through its own `to_json`, a JSON-valued member copied -- and the 34 hand-written `to_json` methods, each an instance of that rule, are gone, along with the 26 in the tests and the one in the door's example. An entity whose JSON is not its fields may still override; none does. What is given up is the per-entity return type. Every `to_json` is typed as the metadata field union rather than as the entity's own TypedDict, so a consumer who wants a member of the written form typed narrows it, as the tests now do through their helpers. The public `*Configuration` TypedDicts stay as the JSON types, and one test ties each to its entity's fields -- same keys, same requiredness -- which nothing did before. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 26 +-- .../src/zarr_metadata/v3/_entity.py | 64 ++++-- .../src/zarr_metadata/v3/_typed_json.py | 202 +++++++++++++++++- .../v3/chunk_grid/rectilinear.py | 6 - .../zarr_metadata/v3/chunk_grid/regular.py | 3 - .../v3/chunk_key_encoding/default.py | 5 - .../zarr_metadata/v3/chunk_key_encoding/v2.py | 5 - .../src/zarr_metadata/v3/codec/blosc.py | 11 - .../src/zarr_metadata/v3/codec/bytes.py | 5 - .../src/zarr_metadata/v3/codec/cast_value.py | 13 -- .../src/zarr_metadata/v3/codec/crc32c.py | 3 - .../src/zarr_metadata/v3/codec/gzip.py | 3 - .../zarr_metadata/v3/codec/scale_offset.py | 13 -- .../v3/codec/sharding_indexed.py | 10 - .../src/zarr_metadata/v3/codec/transpose.py | 3 - .../src/zarr_metadata/v3/codec/zstd.py | 6 - .../src/zarr_metadata/v3/data_type/bool.py | 3 - .../src/zarr_metadata/v3/data_type/bytes.py | 3 - .../zarr_metadata/v3/data_type/complex128.py | 3 - .../zarr_metadata/v3/data_type/complex64.py | 3 - .../src/zarr_metadata/v3/data_type/float16.py | 3 - .../src/zarr_metadata/v3/data_type/float32.py | 3 - .../src/zarr_metadata/v3/data_type/float64.py | 3 - .../src/zarr_metadata/v3/data_type/int16.py | 3 - .../src/zarr_metadata/v3/data_type/int32.py | 3 - .../src/zarr_metadata/v3/data_type/int64.py | 3 - .../src/zarr_metadata/v3/data_type/int8.py | 3 - .../v3/data_type/numpy_datetime64.py | 6 - .../v3/data_type/numpy_timedelta64.py | 6 - .../src/zarr_metadata/v3/data_type/raw.py | 3 - .../src/zarr_metadata/v3/data_type/string.py | 3 - .../src/zarr_metadata/v3/data_type/struct.py | 10 - .../src/zarr_metadata/v3/data_type/uint16.py | 3 - .../src/zarr_metadata/v3/data_type/uint32.py | 3 - .../src/zarr_metadata/v3/data_type/uint64.py | 3 - .../src/zarr_metadata/v3/data_type/uint8.py | 3 - .../src/zarr_metadata/v3/entity.py | 49 +---- .../tests/v3/test_acme_affine.py | 11 +- .../tests/v3/test_acme_decimal.py | 6 - .../zarr-metadata/tests/v3/test_entities.py | 48 +++-- .../tests/v3/test_extension_api.py | 80 +------ 41 files changed, 299 insertions(+), 346 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index fc1709f9cc..97fef360ab 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -96,23 +96,15 @@ registration checks for that are gone. Which of a document's fields holds which kind is the document's own knowledge, in one place; the entity layer no longer names a document field. -`to_json` is abstract: each entity writes its own, as a literal of its -JSON type, which it names as the method's return type -- narrower than -the base's `ZarrV3MetadataFieldJSON` -- and the type checker holds the -literal to that TypedDict, which is what a base method building a -`dict[str, object]` and casting it to the named type could only assert. -The entity base is not generic: nothing is gained by naming the JSON -type twice. The class-creation check that compared the named type with -the fields, the generic `configuration()` and the rendering walk it -needed are gone with the cast; `written` renders a contained field. - -One thing this does not change, under mypy. An entity's JSON type is a -TypedDict, which mypy will not accept where a `ZarrV3MetadataFieldJSON` is -wanted: it reads every TypedDict as `Mapping[str, object]`, never as the -`Mapping[str, JSONValue]` the envelope declares (python/mypy#8994, -python/mypy#18439 -- mypy lacks PEP 728). The conversion is sound and the -annotation stays; a consumer under mypy casts at the one place it puts an -entity's JSON into a document. +`to_json` is written once, in the base, from the fields: `writer_for` +is the parser's inverse over the same annotation, so what `coerce` reads +from a document, `to_json` puts back -- the bare name when every member +is absent, the object otherwise, a contained entity through its own +`to_json`, a JSON-valued member copied. No entity writes its own, and a +third party writes nothing for it. What is given up is the per-entity +return type: every entity's `to_json` is typed as the metadata field +union rather than as its own TypedDict, and a consumer who wants a +member of the written form typed narrows it. From three reviews of the result, each from the vantage of another implementation (zarrs, TensorStore, zarrita.js): every codec declares diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index acb35f72c4..b41ab7efd2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -11,11 +11,10 @@ on the class as `problems`; the constructor stops at the first, `coerce` reports every one. -What an entity writes and what it simplifies to are its own too: -`to_json` is abstract, a literal of the entity's JSON type, and -`canonical` defaults to the entity itself. An entity that contains -entities writes them with `written` and canonicalizes them with -`canonicalized`, in the same two methods. +What an entity writes follows from the same fields: `to_json` is +written once here, the parser's inverse over each field's annotation. +What it simplifies to is its own: `canonical` defaults to the entity +itself, and an entity that contains entities canonicalizes them there. Composition -- what needs the document or the codec chain -- is the entity's to answer through `incoming_problems`, `shape_problems`, @@ -39,6 +38,7 @@ Loc, Parsed, Parser, + Writer, as_tuples, declared_class_vars, field_hints, @@ -49,12 +49,14 @@ parser_for, problem, strip_annotation, + writer, ) if TYPE_CHECKING: from collections.abc import Iterator, Sequence from typing import Self + from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._parts import ArrayParts, ChunkGrid from zarr_metadata.v3._registry import Context @@ -321,6 +323,20 @@ def parse(value: object, loc: Loc, reading: _Reading) -> Parsed: return parse +def _nested_field_writer(annotation: object) -> Writer | None: + """The writer for a field holding another entity: what it holds, as it writes itself.""" + if nested_kind(annotation) is None: + return None + + def write(value: object) -> JSONValue: + if isinstance(value, (MetadataEntity, Opaque)): + return value.to_json() + msg = f"{value!r} is not an entity or an Opaque" + raise TypeError(msg) + + return write + + @dataclass(frozen=True, slots=True) class _Member: """How `coerce` reads one field: what the annotation says, compiled once.""" @@ -329,6 +345,7 @@ class _Member: from_name: bool optional: bool parse: Parser[_Reading] + write: Writer @functools.cache @@ -346,6 +363,7 @@ def _plan(cls: type[MetadataEntity]) -> tuple[_Member, ...]: is_from_name(annotation), is_optional(annotation), parser(annotation, _nested_field), + writer(annotation, _nested_field_writer), ) for key, annotation in field_hints(cls).items() ) @@ -535,25 +553,35 @@ def canonical(self) -> Self: """ return self - @abstractmethod def to_json(self) -> ZarrV3MetadataFieldJSON: - """This entity as a document would write it: a literal of its own JSON type. - - Faithful to every member it holds: read a document, write it + """This entity as a document would write it. + + Written from the fields by the same declaration `coerce` reads + them by, each member by the writer its annotation implies: the + bare name when every member it holds is absent, the object + otherwise, a contained entity through its own `to_json`, a + JSON-valued member copied so the document is not a handle on + the entity. Faithful to every member: read a document, write it back, and those come out as they went in. Ask `canonical` first if you want the simplest equivalent spelling. The envelope's spelling is the one thing not preserved, because the entity does not model it: a bare name, `{"name": x}` and `{"name": x, - "configuration": {}}` all read to the same entity, and the entity - writes the bare name when every member it holds is absent. - - Written per entity, as a literal of its own TypedDict and with - that TypedDict as the declared return type -- narrower than the - base's, which is what tells a consumer holding a `GzipCodec` that - it gets a `GzipCodecObject` -- so pyright checks the literal's keys - and values against it. A contained entity is written with - `written`. + "configuration": {}}` all read to the same entity. + + An entity whose JSON is not its fields overrides this; none in + the package does. """ + name = self.identifier + configuration: dict[str, JSONValue] = {} + for member in _plan(type(self)): + value = getattr(self, member.key) + if member.from_name: + name = value if isinstance(value, str) else name + elif value is not UNSET: + configuration[member.key] = member.write(value) + if len(configuration) == 0: + return name + return {"name": name, "configuration": configuration} @dataclass(frozen=True) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index 9d4e25aacc..e482e4d0a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -1,9 +1,11 @@ """JSON values parsed by type annotation. -A dataclass's fields are its schema, and this module reads that schema. -`parser_for` turns a field annotation into a parser: a function of a -JSON value and its location that returns the typed value and every -problem found with it. The annotations it reads are the shapes JSON +A dataclass's fields are its schema, and this module reads that schema +in both directions. `parser_for` turns a field annotation into a parser: +a function of a JSON value and its location that returns the typed +value and every problem found with it. `writer_for` turns the same +annotation into the parser's inverse: a function of the typed value that +returns the JSON a document writes for it. The annotations they read are the shapes JSON takes and no others -- `int`, `float` for any number, `bool`, `str`, `JSONValue`, a `Literal` of names, `tuple[T, ...]` and `tuple[T1, T2]`, a union of those, an object described by a TypedDict or a record @@ -26,6 +28,7 @@ import sys import types from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy from dataclasses import is_dataclass from typing import ( TYPE_CHECKING, @@ -75,6 +78,12 @@ Leaf: TypeAlias = Callable[[object], "Parser[S] | None"] """A caller's own shapes: asked first for every annotation, None to decline.""" +Writer: TypeAlias = Callable[[object], JSONValue] +"""A typed value as the JSON a document writes for it: a parser's inverse over the same annotation.""" + +WriterLeaf: TypeAlias = Callable[[object], "Writer | None"] +"""A caller's own shapes, for writing: asked first for every annotation, None to decline.""" + def problem( loc: Loc, message: str, kind: ProblemKind = "invalid_type" @@ -650,17 +659,195 @@ def parser(annotation: object, leaf: Leaf[S]) -> Parser[S]: return found +# --- the writers --------------------------------------------------------- +# +# The inverse of each parser, over the same annotation. A writer is asked +# for a value the entity holds, which its field's type vouches for; the +# checks here are what stand between a hand-built entity holding +# something that is not JSON and a document that is not JSON. + + +def _not_json(value: object) -> TypeError: + return TypeError( + f"{value!r} is not a JSON value; an entity's members are the JSON the document writes" + ) + + +def _as_json(value: object) -> JSONValue: + """A scalar or a JSON value as it is.""" + if is_json(value): + return value + raise _not_json(value) + + +def _copied(value: object) -> JSONValue: + """A JSON value, copied: the document handed out is not a handle on a frozen entity.""" + if is_json(value): + return deepcopy(value) + raise _not_json(value) + + +def each_of(element: Writer) -> Writer: + """An array, each element written by its type.""" + + def write(value: object) -> JSONValue: + if not isinstance(value, (list, tuple)): + raise _not_json(value) + return tuple(element(entry) for entry in cast("list[object] | tuple[object, ...]", value)) + + return write + + +def positions_of(elements: Sequence[Writer]) -> Writer: + """An array of a fixed length, each position written by its type.""" + + def write(value: object) -> JSONValue: + if not isinstance(value, (list, tuple)): + raise _not_json(value) + entries = cast("list[object] | tuple[object, ...]", value) + return tuple(element(entry) for element, entry in zip(elements, entries, strict=True)) + + return write + + +def one_of_writers(branches: Sequence[tuple[object, Writer]]) -> Writer: + """A union, written by the branch whose shape the value has, as the parser chose it.""" + + def write(value: object) -> JSONValue: + for annotation, branch in branches: + if has_shape(shape_of(annotation), value): + return branch(value) + return _as_json(value) + + return write + + +def keys_of(members: Mapping[str, Writer]) -> Writer: + """An object kept as a mapping: declared keys written by their types, the rest copied.""" + + def write(value: object) -> JSONValue: + if not isinstance(value, Mapping): + raise _not_json(value) + entries = cast("Mapping[str, object]", value) + return { + key: members[key](entry) if key in members else _copied(entry) + for key, entry in entries.items() + } + + return write + + +def fields_of(members: Mapping[str, Writer]) -> Writer: + """A record dataclass as an object: each field written by its type, an absent optional one left out.""" + + def write(value: object) -> JSONValue: + written: dict[str, JSONValue] = {} + for key, member in members.items(): + entry = getattr(value, key) + if entry is not UNSET: + written[key] = member(entry) + return written + + return write + + +def values_of(value: Writer) -> Writer: + """An object of any keys, each value written by its type.""" + + def write(candidate: object) -> JSONValue: + if not isinstance(candidate, Mapping): + raise _not_json(candidate) + entries = cast("Mapping[str, object]", candidate) + return {key: value(entry) for key, entry in entries.items()} + + return write + + +def _writers_of(annotations: Mapping[str, object], leaf: WriterLeaf) -> dict[str, Writer] | None: + members: dict[str, Writer] = {} + for key, annotation in annotations.items(): + member = writer_for(annotation, leaf) + if member is None: + return None + members[key] = member + return members + + +def no_writer_leaf(annotation: object) -> Writer | None: + """The leaf of a caller with no shapes of its own.""" + return None + + +def writer_for(annotation: object, leaf: WriterLeaf) -> Writer | None: + """The writer a field annotation implies, or None if it implies none. + + The inverse of `parser_for` over the same shapes: what the parser + reads from a document, the writer puts back. `leaf` is asked first, + here and at every depth, as the parser's is. + """ + inner = without_unset(strip_annotation(annotation)[0]) + found = leaf(inner) + if found is not None: + return found + if inner is int or inner is float or inner is bool or inner is str: + return _as_json + if get_origin(inner) is Literal: + return _as_json + if inner is JSONValue: + return _copied + if is_union(inner): + compiled = [(branch, writer_for(branch, leaf)) for branch in get_args(inner)] + branches = [(branch, member) for branch, member in compiled if member is not None] + return one_of_writers(branches) if len(branches) == len(compiled) else None + if get_origin(inner) is tuple: + arguments = get_args(inner) + if len(arguments) == 2 and arguments[1] is Ellipsis: + element = writer_for(arguments[0], leaf) + return None if element is None else each_of(element) + compiled = [writer_for(argument, leaf) for argument in arguments] + elements = [element for element in compiled if element is not None] + return positions_of(elements) if len(elements) == len(compiled) else None + if is_typeddict(inner): + members = _writers_of(get_type_hints(inner, include_extras=True), leaf) + return None if members is None else keys_of(members) + if get_origin(inner) in (Mapping, dict): + arguments = get_args(inner) + if len(arguments) != 2 or arguments[0] is not str: + return None + value = writer_for(arguments[1], leaf) + return None if value is None else values_of(value) + if isinstance(inner, NewType): + return writer_for(inner.__supertype__, leaf) + if isinstance(inner, type) and is_dataclass(inner): + members = _writers_of(field_hints(inner), leaf) + return None if members is None else fields_of(members) + return None + + +def writer(annotation: object, leaf: WriterLeaf) -> Writer: + """The writer a field annotation implies; `TypeError` if it implies none.""" + found = writer_for(annotation, leaf) + if found is None: + msg = f"{annotation!r} is not a shape JSON takes" + raise TypeError(msg) + return found + + __all__ = [ "Leaf", "Loc", "Members", "Parsed", "Parser", + "Writer", + "WriterLeaf", "any_of", "as_tuples", "declared_class_vars", "describe", + "each_of", "field_hints", + "fields_of", "fixed_tuple", "has_shape", "is_class_var", @@ -668,17 +855,24 @@ def parser(annotation: object, leaf: Leaf[S]) -> Parser[S]: "is_not_required", "is_optional", "is_union", + "keys_of", "mapping_of", "no_leaf", + "no_writer_leaf", "object_of", "one_of", + "one_of_writers", "own_annotations", "parser", "parser_for", + "positions_of", "problem", "record_of", "sequence_of", "shape_of", "strip_annotation", + "values_of", "without_unset", + "writer", + "writer_for", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 74457916bc..c9a80a49a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -242,9 +242,3 @@ def canonical(self) -> Self: grid, and the encoded one stays the same size as the array grows. """ return replace(self, chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) - - def to_json(self) -> RectilinearChunkGridObject: - return { - "name": "rectilinear", - "configuration": {"kind": self.kind, "chunk_shapes": self.chunk_shapes}, - } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index a59f376bb5..1310852fb7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -95,6 +95,3 @@ def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: def grid(self, array_shape: object) -> ChunkGrid: """One extent per axis, the same for every chunk on that axis.""" return ChunkGrid.regular(self.chunk_shape) - - def to_json(self) -> RegularChunkGridObject: - return {"name": "regular", "configuration": {"chunk_shape": self.chunk_shape}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 9eeabbf75f..2943d7e6b5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -77,8 +77,3 @@ class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity): separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME - - def to_json(self) -> DefaultChunkKeyEncodingObject | DefaultChunkKeyEncodingName: - if self.separator is UNSET: - return "default" - return {"name": "default", "configuration": {"separator": self.separator}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index d0182ffb87..d41fbbcf1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -83,8 +83,3 @@ class V2ChunkKeyEncoding(ChunkKeyEncodingEntity): separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME - - def to_json(self) -> V2ChunkKeyEncodingObject | V2ChunkKeyEncodingName: - if self.separator is UNSET: - return "v2" - return {"name": "v2", "configuration": {"separator": self.separator}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 90f26d3297..7d1096faf5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -150,14 +150,3 @@ def canonical(self) -> Self: if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: return self return replace(self, typesize=UNSET) - - def to_json(self) -> BloscCodecObject: - configuration: BloscCodecConfiguration = { - "cname": self.cname, - "clevel": self.clevel, - "shuffle": self.shuffle, - "blocksize": self.blocksize, - } - if self.typesize is not UNSET: - configuration["typesize"] = self.typesize - return {"name": "blosc", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 30d66137ac..c1753db981 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -118,8 +118,3 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb "missing_key", ) return () - - def to_json(self) -> BytesCodecObject | BytesCodecName: - if self.endian is UNSET: - return "bytes" - return {"name": "bytes", "configuration": {"endian": self.endian}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index fc7ad47b75..fd65f622ce 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -4,7 +4,6 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ -from copy import deepcopy from dataclasses import dataclass, replace from typing import ClassVar, Final, Literal, NotRequired, Self @@ -149,15 +148,3 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" data_type = self.data_type return incoming.with_data_type(data_type if isinstance(data_type, DataTypeEntity) else None) - - def to_json(self) -> CastValueCodecObject: - configuration: CastValueCodecConfiguration = {"data_type": self.data_type.to_json()} - if self.rounding is not UNSET: - configuration["rounding"] = self.rounding - if self.out_of_range is not UNSET: - configuration["out_of_range"] = self.out_of_range - if self.scalar_map is not UNSET: - # Copied: the document handed out must not be a handle on - # this frozen entity's own map. - configuration["scalar_map"] = deepcopy(self.scalar_map) - return {"name": "cast_value", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 1ed9917444..963734141e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -68,6 +68,3 @@ class Crc32cCodec(BytesBytesCodec): identifier: ClassVar[str] = CRC32C_CODEC_NAME variable_size: ClassVar[bool] = False - - def to_json(self) -> Crc32cCodecName: - return "crc32c" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 3acc707e9d..ec17ba640e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -84,6 +84,3 @@ class GzipCodec(BytesBytesCodec): variable_size: ClassVar[bool] = True problems = gzip_problems - - def to_json(self) -> GzipCodecObject: - return {"name": "gzip", "configuration": {"level": self.level}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index fe313872d9..f51fd13aa0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -4,7 +4,6 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ -from copy import deepcopy from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired @@ -113,15 +112,3 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: longer changes the element type -- only the values. """ return incoming - - def to_json(self) -> ScaleOffsetCodecObject | ScaleOffsetCodecName: - configuration: ScaleOffsetCodecConfiguration = {} - # Copied: a member may be a JSON object, and the document handed - # out must not be a handle on this frozen entity. - if self.offset is not UNSET: - configuration["offset"] = deepcopy(self.offset) - if self.scale is not UNSET: - configuration["scale"] = deepcopy(self.scale) - if len(configuration) == 0: - return "scale_offset" - return {"name": "scale_offset", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 257671b331..02634c2f14 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -203,13 +203,3 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation ) ) return tuple(found) - - def to_json(self) -> ShardingIndexedCodecObject: - configuration: ShardingIndexedCodecConfiguration = { - "chunk_shape": self.chunk_shape, - "codecs": tuple(codec.to_json() for codec in self.codecs), - "index_codecs": tuple(codec.to_json() for codec in self.index_codecs), - } - if self.index_location is not UNSET: - configuration["index_location"] = self.index_location - return {"name": "sharding_indexed", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 86c4feac7e..cbf7af3c62 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -113,6 +113,3 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: longer the grid the document wrote. """ return incoming.with_grid(incoming.grid.permuted(self.order)) - - def to_json(self) -> TransposeCodecObject: - return {"name": "transpose", "configuration": {"order": self.order}} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 2b04176a30..63f7d25d39 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -95,9 +95,3 @@ class ZstdCodec(BytesBytesCodec): variable_size: ClassVar[bool] = True problems = zstd_problems - - def to_json(self) -> ZstdCodecObject: - configuration: ZstdCodecConfiguration = {"level": self.level} - if self.checksum is not UNSET: - configuration["checksum"] = self.checksum - return {"name": "zstd", "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index 4ac5e5cbb2..d240ffdf2d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -44,6 +44,3 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP if not isinstance(value, bool): return problem(loc, f"expected a boolean, got {value!r}", "invalid_value") return () - - def to_json(self) -> BoolDataTypeName: - return "bool" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index 3e1107ae0a..c52e5237e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -77,6 +77,3 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP ) return () return byte_values(value, None, loc) - - def to_json(self) -> BytesDataTypeName: - return "bytes" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py index 420aec8caa..12ff217385 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex128.py @@ -48,6 +48,3 @@ class Complex128DataType(ComplexDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" component: ClassVar[type[FloatDataType]] = Float64DataType identifier: ClassVar[str] = COMPLEX128_DATA_TYPE_NAME - - def to_json(self) -> Complex128DataTypeName: - return "complex128" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py index 1e85f0d244..cf06a9214d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/complex64.py @@ -48,6 +48,3 @@ class Complex64DataType(ComplexDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" component: ClassVar[type[FloatDataType]] = Float32DataType identifier: ClassVar[str] = COMPLEX64_DATA_TYPE_NAME - - def to_json(self) -> Complex64DataTypeName: - return "complex64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index d537fe9611..d6504868d4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -88,6 +88,3 @@ class Float16DataType(FloatDataType): hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float16) largest: ClassVar[float | None] = 65504.0 identifier: ClassVar[str] = FLOAT16_DATA_TYPE_NAME - - def to_json(self) -> Float16DataTypeName: - return "float16" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 4386750b21..5e2b287d52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -88,6 +88,3 @@ class Float32DataType(FloatDataType): hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float32) largest: ClassVar[float | None] = 3.4028235e38 identifier: ClassVar[str] = FLOAT32_DATA_TYPE_NAME - - def to_json(self) -> Float32DataTypeName: - return "float32" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index aad3a99daf..b52b9205bd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -89,6 +89,3 @@ class Float64DataType(FloatDataType): hex_parser: ClassVar[Callable[[str], object]] = staticmethod(hex_float64) largest: ClassVar[float | None] = None identifier: ClassVar[str] = FLOAT64_DATA_TYPE_NAME - - def to_json(self) -> Float64DataTypeName: - return "float64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py index 6911ca69cc..fb295e3dfc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int16.py @@ -35,6 +35,3 @@ class Int16DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (-32768, 32767) identifier: ClassVar[str] = INT16_DATA_TYPE_NAME - - def to_json(self) -> Int16DataTypeName: - return "int16" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py index d900e1b18e..7d35f033cd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int32.py @@ -35,6 +35,3 @@ class Int32DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (-2147483648, 2147483647) identifier: ClassVar[str] = INT32_DATA_TYPE_NAME - - def to_json(self) -> Int32DataTypeName: - return "int32" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py index f76d5a9de2..365370a9dc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int64.py @@ -35,6 +35,3 @@ class Int64DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (-9223372036854775808, 9223372036854775807) identifier: ClassVar[str] = INT64_DATA_TYPE_NAME - - def to_json(self) -> Int64DataTypeName: - return "int64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py index 56780227e8..5a2bf185ca 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/int8.py @@ -35,6 +35,3 @@ class Int8DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "single_byte" bounds: ClassVar[tuple[int, int]] = (-128, 127) identifier: ClassVar[str] = INT8_DATA_TYPE_NAME - - def to_json(self) -> Int8DataTypeName: - return "int8" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 55280e4f59..cd6f75786e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -72,9 +72,3 @@ class NumpyDatetime64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_DATETIME64_DATA_TYPE_NAME - - def to_json(self) -> NumpyDatetime64: - return { - "name": "numpy.datetime64", - "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, - } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index 2eb50ffba2..46b50b003e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -76,9 +76,3 @@ class NumpyTimedelta64DataType(NumpyTimeDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" identifier: ClassVar[str] = NUMPY_TIMEDELTA64_DATA_TYPE_NAME - - def to_json(self) -> NumpyTimedelta64: - return { - "name": "numpy.timedelta64", - "configuration": {"unit": self.unit, "scale_factor": self.scale_factor}, - } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index ee6b2e671f..13db2e357e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -128,9 +128,6 @@ def accepts(cls, name: str) -> bool: problems = raw_bytes_problems - def to_json(self) -> RawBytesDataTypeName: - return RawBytesDataTypeName(self.data_type_name) - def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """One byte value per byte of the scalar. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 0ce4e9351c..826273607c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -44,6 +44,3 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP if not isinstance(value, str): return problem(loc, f"expected a string, got {value!r}", "invalid_value") return () - - def to_json(self) -> StringDataTypeName: - return "string" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 7219797838..852dec071e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -98,10 +98,6 @@ class StructFieldComponent: data_type: DataTypeEntity | Opaque -def _written_field(field: StructFieldComponent) -> StructField: - return {"name": field.name, "data_type": field.data_type.to_json()} - - def struct_problems(data_type: "StructDataType", /) -> "Iterator[ValidationProblem]": """Names exist, are non-empty and distinct; types are fixed-size. @@ -214,9 +210,3 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP for key in sorted(fills.keys() - declared) ) return tuple(found) - - def to_json(self) -> Struct: - return { - "name": "struct", - "configuration": {"fields": tuple(_written_field(field) for field in self.fields)}, - } diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py index 0bbcd7c603..d87340123c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint16.py @@ -35,6 +35,3 @@ class Uint16DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (0, 65535) identifier: ClassVar[str] = UINT16_DATA_TYPE_NAME - - def to_json(self) -> Uint16DataTypeName: - return "uint16" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py index bce64cd4ca..ab0c7b2cce 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint32.py @@ -35,6 +35,3 @@ class Uint32DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (0, 4294967295) identifier: ClassVar[str] = UINT32_DATA_TYPE_NAME - - def to_json(self) -> Uint32DataTypeName: - return "uint32" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py index 5948ee211d..241f4eb819 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint64.py @@ -35,6 +35,3 @@ class Uint64DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "multi_byte" bounds: ClassVar[tuple[int, int]] = (0, 18446744073709551615) identifier: ClassVar[str] = UINT64_DATA_TYPE_NAME - - def to_json(self) -> Uint64DataTypeName: - return "uint64" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py index f5a68d2a80..fb5fabbf5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/uint8.py @@ -35,6 +35,3 @@ class Uint8DataType(IntegerDataType): scalar_storage: ClassVar[StorageClass] = "single_byte" bounds: ClassVar[tuple[int, int]] = (0, 255) identifier: ClassVar[str] = UINT8_DATA_TYPE_NAME - - def to_json(self) -> Uint8DataTypeName: - return "uint8" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 12913f208e..82784f3fd0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -46,9 +46,7 @@ from collections.abc import Iterator from dataclasses import dataclass - from typing import ClassVar, Literal, NotRequired - - from typing_extensions import TypedDict + from typing import ClassVar from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3.entity import ( @@ -58,13 +56,6 @@ ValidationProblem, ) - class AcmeLz4Configuration(TypedDict, closed=True): - acceleration: NotRequired[int] - - class AcmeLz4Object(TypedDict, closed=True): - name: Literal["acme.lz4"] - configuration: AcmeLz4Configuration - def acme_lz4_problems(codec: "AcmeLz4Codec", /) -> Iterator[ValidationProblem]: if codec.acceleration is not UNSET and not 1 <= codec.acceleration <= 65537: yield ValidationProblem( @@ -81,25 +72,21 @@ class AcmeLz4Codec(BytesBytesCodec): variable_size: ClassVar[bool] = True # a compressor: its output length is not fixed problems = acme_lz4_problems - def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: - if self.acceleration is UNSET: - return "acme.lz4" - return {"name": "acme.lz4", "configuration": {"acceleration": self.acceleration}} - SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec) validate_array_metadata_v3(document, context=SCOPE) The fields are the only place the shape is written. Which members exist, -which may be left out (the type admits `UNSET`), and how each one is -type-checked are all read off the annotations, and the shapes are the +which may be left out (the type admits `UNSET`), how each one is +type-checked, and how each is written back are all read off the +annotations, and the shapes are the ones JSON takes: `int`, `float` (any JSON number), `bool`, `str`, `JSONValue`, a `Literal` of names, `tuple[T, ...]` or `tuple[T1, T2]`, a TypedDict or dataclass record, `Mapping[str, V]`, a `NewType`, and a nested entity, always as `inner: CodecEntity | Opaque`, because that is what the field holds when the inner name is out of scope -- at any depth, as an array's element or a record's field, and read in the scope -the containing entity is read in. Anything else is refused at class -creation. A required member +the containing entity is read in. Anything else is refused at +registration. A required member has no default; an optional one is `| UNSET = UNSET`, so absence stays distinct from a JSON `null`, and a member that means something when absent is read that way where it is used, not defaulted. @@ -117,11 +104,11 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: It runs only on an entity whose members all read: a member of the wrong type is reported and the entity is not built. -**What an entity answers for itself**, beyond its fields. `to_json`, -abstract: the entity as a document writes it, as a literal of its own -TypedDict, which pyright holds to that type -- the bare name when every +**What an entity answers for itself**, beyond its fields. `to_json` is +written once in the base, from the fields: the bare name when every member is absent, the object otherwise, a contained entity through its -own `to_json`. `canonical`, the entity in its simplest equivalent form: +own `to_json`; an entity whose JSON is not its fields overrides it, and +none in the package does. `canonical`, the entity in its simplest equivalent form: the entity itself by default, overridden where two spellings of its members mean the same, and in an entity that contains entities to put those in canonical form -- `replace(self, inner=self.inner.canonical())`. @@ -164,13 +151,6 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: base, its key is its `identifier`, so `extended_with` takes the classes and nothing can be misfiled. -**Naming the JSON type.** The return annotation of `to_json` -- above, -`AcmeLz4Object | Literal["acme.lz4"]` -- is the entity's own JSON type, -narrower than the `ZarrV3MetadataFieldJSON` the base declares, and -pyright checks the literal returned against it: a key it does not -declare, a required one left out, a value of the wrong type is a static -error. - Two complete extensions written against this module alone, as tests: `tests/v3/test_acme_affine.py` (an `array_array` codec with a number, an optional member and a nested data type) and @@ -181,15 +161,6 @@ def to_json(self) -> AcmeLz4Object | Literal["acme.lz4"]: means -- so registering yours is how you get it judged rather than waved through. `CORE` is what the specification defines; `CORE_AND_EXTENSIONS` adds the `zarr-extensions` registry; `extended_with(*classes)` adds yours. - -One known friction, under mypy only. An entity's `to_json` returns its -own object TypedDict, and mypy does not accept that where a -`ZarrV3MetadataFieldJSON` is wanted: it reads every TypedDict as -`Mapping[str, object]`, never as the `Mapping[str, JSONValue]` the -envelope declares (python/mypy#8994, python/mypy#18439 -- mypy lacks -PEP 728, which every TypedDict here relies on). The conversion is sound -and the annotation stays; a consumer under mypy casts at the one place -it puts an entity's JSON into a document. Pyright accepts it. """ from __future__ import annotations diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 01d75cc10c..bbbd1813a6 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -81,14 +81,6 @@ def canonical(self) -> Self: dtype=UNSET if self.dtype is UNSET else self.dtype.canonical(), ) - def to_json(self) -> AcmeAffineObject: - configuration: AcmeAffineConfiguration = {"scale": self.scale} - if self.offset is not UNSET: - configuration["offset"] = self.offset - if self.dtype is not UNSET: - configuration["dtype"] = self.dtype.to_json() - return {"name": "acme.affine", "configuration": configuration} - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: data_type = incoming.data_type if incoming is not None else None if data_type is None or data_type.storage_class() != "variable_length": @@ -204,8 +196,7 @@ def test_round_trip_and_canonical() -> None: entry = _affine(scale=2, offset=1, dtype="float64") codec = ArrayDocumentV3.from_json(_document(codecs=[entry, BYTES_LE]), context=SCOPE).codecs[0] assert isinstance(codec, AcmeAffineCodec) - written: AcmeAffineObject = codec.to_json() - assert written == entry + assert codec.to_json() == entry assert AcmeAffineCodec(scale=2, offset=0).canonical() == AcmeAffineCodec(scale=2) document = _document(codecs=[_affine(scale=2, offset=0.0), BYTES_LE]) result = canonicalize_array_metadata_v3(document, context=SCOPE) diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 96c08b3130..32374cfa0a 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -100,12 +100,6 @@ class AcmeDecimalDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "multi_byte" problems = acme_decimal_problems - def to_json(self) -> AcmeDecimal: - return { - "name": "acme.decimal", - "configuration": {"precision": self.precision, "scale": self.scale}, - } - def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """A decimal literal whose digits fit `precision` and `scale`. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 4b8b9b8a61..c8a0c41546 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,27 +11,27 @@ import copy import dataclasses +import sys from typing import ( - TYPE_CHECKING, Any, ClassVar, Self, cast, + get_type_hints, ) import pytest from hypothesis import given, settings +from typing_extensions import is_typeddict -from tests.helpers import configuration_of +from tests.helpers import configuration_of, entry_at from tests.rules.strategies import valid_documents - -if TYPE_CHECKING: - from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON - from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._document import read_array_v3 +from zarr_metadata.v3._entity import is_from_name from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS +from zarr_metadata.v3._typed_json import field_hints, is_not_required, is_optional from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, ) @@ -606,7 +606,7 @@ def test_canonical_is_what_simplifies() -> None: }, ) assert isinstance(blosc, BloscCodec) - assert "typesize" not in blosc.canonical().to_json()["configuration"] + assert "typesize" not in configuration_of(blosc.canonical().to_json()) def test_canonical_reaches_a_contained_entity() -> None: @@ -634,7 +634,7 @@ def test_canonical_reaches_a_contained_entity() -> None: }, ) assert isinstance(shard, ShardingIndexedCodec) - inner = shard.canonical().to_json()["configuration"]["codecs"][1] + inner = entry_at(shard.canonical().to_json(), "configuration", "codecs", 1) assert "typesize" not in configuration_of(inner) @@ -711,7 +711,7 @@ def test_a_member_the_entity_does_not_model_is_not_written_back() -> None: codec, problems = CORE_AND_EXTENSIONS.coerce(CodecEntity, entry) assert [(p.loc, p.kind) for p in problems] == [(("configuration", "typo_key"), "unknown_key")] assert isinstance(codec, BloscCodec) - assert "typo_key" not in codec.to_json()["configuration"] + assert "typo_key" not in configuration_of(codec.to_json()) def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: @@ -747,11 +747,6 @@ class AcmeShardCache(StorageTransformerEntity): def canonical(self) -> Self: return dataclasses.replace(self, verbose=UNSET) - def to_json(self) -> ZarrV3MetadataFieldJSON: - if self.verbose is UNSET: - return "acme.shard_cache" - return {"name": "acme.shard_cache", "configuration": {"verbose": self.verbose}} - def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> None: # `to_json` is faithful, entities included; `canonical` walks every @@ -794,3 +789,28 @@ def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> ) assert canonical["storage_transformers"] == ("acme.shard_cache",) assert "dimension_names" not in canonical + + +def test_the_fields_are_the_public_configuration_type() -> None: + # Each entity module's `*Configuration` TypedDict is the public JSON + # type of what the entity holds; the fields are what it reads and + # writes. Nothing else ties the two, so this does: same keys, same + # requiredness. An entity of no members has no such type. + for cls in CORE_AND_EXTENSIONS.entities(): + hints = field_hints(cls) + members = {key for key, annotation in hints.items() if not is_from_name(annotation)} + required = {key for key in members if not is_optional(hints[key])} + declared = [ + value + for name, value in vars(sys.modules[cls.__module__]).items() + if name.endswith("Configuration") and is_typeddict(value) + ] + if len(declared) == 0: + assert members == set(), cls + continue + (configuration,) = declared + keys = get_type_hints(configuration, include_extras=True) + assert set(keys) == members, cls + assert { + key for key, annotation in keys.items() if not is_not_required(annotation) + } == required, cls diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index eaee235d59..f77440b7fd 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,7 +8,7 @@ import re from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self, cast +from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self import pytest from typing_extensions import TypedDict @@ -33,13 +33,11 @@ Context, DataTypeEntity, IntegerDataType, - JSONValue, Loc, MetadataEntity, Opaque, StorageClass, ValidationProblem, - ZarrV3MetadataFieldJSON, problem, ) @@ -68,11 +66,6 @@ class AcmeLz4Codec(BytesBytesCodec): variable_size: ClassVar[bool] = True problems = acme_lz4_problems - def to_json(self) -> ZarrV3MetadataFieldJSON: - if self.acceleration is UNSET: - return "acme.lz4" - return {"name": "acme.lz4", "configuration": {"acceleration": self.acceleration}} - @dataclass(frozen=True) class AcmeFloat8DataType(DataTypeEntity): @@ -84,9 +77,6 @@ class AcmeFloat8DataType(DataTypeEntity): def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: return () - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.float8" - def _scope() -> Context: return CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec, AcmeFloat8DataType) @@ -166,9 +156,6 @@ class Nameless(BytesBytesCodec): variable_size: ClassVar[bool] = True - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.nameless" - with pytest.raises(TypeError, match="does not declare identifier"): CORE_AND_EXTENSIONS.extended_with(Nameless) @@ -201,11 +188,6 @@ class Defaulted(BytesBytesCodec): variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - if self.level is UNSET: - return "acme.defaulted" - return {"name": "acme.defaulted", "configuration": {"level": self.level}} - assert Defaulted().level == 3 codec, problems = CORE_AND_EXTENSIONS.extended_with(Defaulted).coerce( CodecEntity, "acme.defaulted" @@ -284,9 +266,6 @@ def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: class Int24DataType(IntegerDataType): identifier: ClassVar[str] = "acme.int24" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.int24" - with pytest.raises(TypeError, match="does not declare bounds"): CORE_AND_EXTENSIONS.extended_with(Int24DataType) @@ -318,9 +297,6 @@ class AcmeFixedDataType(DataTypeEntity): def accepts(cls, name: str) -> bool: return ACME_FIXED_PATTERN.fullmatch(name) is not None - def to_json(self) -> ZarrV3MetadataFieldJSON: - return cast("ZarrV3MetadataFieldJSON", self.data_type_name) - def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: return () @@ -360,9 +336,6 @@ class Structured(BytesBytesCodec): identifier: ClassVar[str] = "acme.structured" variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.structured" - with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): CORE_AND_EXTENSIONS.extended_with(Structured) @@ -381,9 +354,6 @@ class AcmeWrapperCodec(BytesBytesCodec): def canonical(self) -> Self: return replace(self, inner=self.inner.canonical()) - def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.wrapper", "configuration": {"inner": self.inner.to_json()}} - def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: # `inner: CodecEntity | Opaque` is the whole declaration of the @@ -464,12 +434,6 @@ def canonical(self) -> Self: frame=UNSET if self.frame == 0 else self.frame, ) - def to_json(self) -> ZarrV3MetadataFieldJSON: - configuration: dict[str, JSONValue] = {"inner": self.inner.to_json()} - if self.frame is not UNSET: - configuration["frame"] = self.frame - return {"name": "acme.framed", "configuration": configuration} - blosc = BloscCodec(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) framed = AcmeFramedCodec(inner=blosc, frame=0) assert framed.canonical() == AcmeFramedCodec(inner=replace(blosc, typesize=UNSET)) @@ -486,9 +450,6 @@ class Vague(BytesBytesCodec): identifier: ClassVar[str] = "acme.vague" variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.vague" - with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): CORE_AND_EXTENSIONS.extended_with(Vague) @@ -534,9 +495,6 @@ class AcmeBlockCodec(BytesBytesCodec): variable_size: ClassVar[bool] = False problems = acme_block_problems - def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.block", "configuration": {"block": self.block}} - def test_a_rule_about_a_member_is_a_function_of_the_instance() -> None: # The rule runs on the typed members and reports relative to the @@ -584,9 +542,6 @@ class AcmeRangeCodec(BytesBytesCodec): variable_size: ClassVar[bool] = False problems = acme_range_problems - def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.range", "configuration": {"low": self.low, "high": self.high}} - def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: # One function, two consumers: the constructor takes the first @@ -618,9 +573,6 @@ class Checked(BytesBytesCodec): def __post_init__(self) -> None: return None - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.checked" - with pytest.raises(TypeError, match="defines __post_init__; write its rules as a function"): CORE_AND_EXTENSIONS.extended_with(Checked) @@ -638,9 +590,6 @@ class Localized(BytesBytesCodec): identifier: ClassVar[str] = "acme.localized" variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.localized" - with pytest.raises(TypeError, match="a field annotation names 'Local', which is not defined"): CORE_AND_EXTENSIONS.extended_with(Localized) @@ -652,9 +601,6 @@ def test_error_a_codec_says_whether_its_output_size_is_fixed() -> None: class Sizeless(BytesBytesCodec): identifier: ClassVar[str] = "acme.sizeless" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.sizeless" - with pytest.raises(TypeError, match="does not declare variable_size"): CORE_AND_EXTENSIONS.extended_with(Sizeless) @@ -687,9 +633,6 @@ class AcmeSlotted(BytesBytesCodec): variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.slotted", "configuration": {"level": self.level}} - assert AcmeSlotted(level=1).to_json() == { "name": "acme.slotted", "configuration": {"level": 1}, @@ -703,9 +646,6 @@ class AcmeNoted(BytesBytesCodec): variable_size: ClassVar[bool] = False note: ClassVar = "not a member" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.noted" - assert AcmeNoted().to_json() == "acme.noted" @@ -724,9 +664,6 @@ class AcmeScaled(ArrayArrayCodec): def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming - def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.scaled", "configuration": {"scale": self.scale}} - scope = CORE_AND_EXTENSIONS.extended_with(AcmeScaled) for spelled in (2, 2.5): codec, problems = scope.coerce( @@ -751,9 +688,6 @@ class Undecorated(BytesBytesCodec): identifier: ClassVar[str] = "acme.undecorated" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return {"name": "acme.undecorated", "configuration": {"level": self.level}} - with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): CORE_AND_EXTENSIONS.extended_with(Undecorated) @@ -767,9 +701,6 @@ class Closed(BytesBytesCodec): identifier: ClassVar[str] = "acme.closed" variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.closed" - with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): CORE_AND_EXTENSIONS.extended_with(Closed) @@ -782,9 +713,6 @@ class Silent(ArrayArrayCodec): identifier: ClassVar[str] = "acme.silent" variable_size: ClassVar[bool] = False - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.silent" - with pytest.raises( TypeError, match="does not define transition, which its base leaves abstract" ): @@ -798,9 +726,6 @@ def test_error_a_codec_is_of_a_kind() -> None: class Kindless(CodecEntity): identifier: ClassVar[str] = "acme.kindless" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.kindless" - with pytest.raises( TypeError, match="subclasses CodecEntity directly; subclass ArrayArrayCodec" ): @@ -814,9 +739,6 @@ class Lax(DataTypeEntity): identifier: ClassVar[str] = "acme.lax" scalar_storage: ClassVar[StorageClass] = "single_byte" - def to_json(self) -> ZarrV3MetadataFieldJSON: - return "acme.lax" - with pytest.raises(TypeError, match="does not define fill_value_problems"): CORE_AND_EXTENSIONS.extended_with(Lax) From db3beae97c7d82585034d543bc7aebec78ae9dff Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 21:27:14 +0200 Subject: [PATCH 090/107] refactor(zarr-metadata): an entity is a name and a configuration record The metadata is `{name, configuration}`, and the entity now has that shape: the name is the class, and its one field, `configuration`, names a frozen dataclass of the members. The lift of configuration keys to the entity, which nothing in Python expressed, is gone with the per-member loop in `coerce` that did it: the configuration is parsed against the record by the parser's record shape, and written back by the writer's. Each entity lifts its members back to the top level with a `@property` per member, and `with_configuration(**changes)` is the entity with members of its configuration replaced, checked as any construction is. Hand construction reads `GzipCodec(GzipOptions(level=5))`. Every record is named `Options`, since `Configuration` is the public JSON TypedDict; the name is a placeholder. The 40,000-document differential is unchanged to the problem. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_entity.py | 267 ++++++++++-------- .../src/zarr_metadata/v3/_typed_json.py | 18 +- .../v3/chunk_grid/rectilinear.py | 23 +- .../zarr_metadata/v3/chunk_grid/regular.py | 13 +- .../v3/chunk_key_encoding/default.py | 13 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 13 +- .../src/zarr_metadata/v3/codec/blosc.py | 41 ++- .../src/zarr_metadata/v3/codec/bytes.py | 13 +- .../src/zarr_metadata/v3/codec/cast_value.py | 35 ++- .../src/zarr_metadata/v3/codec/gzip.py | 13 +- .../zarr_metadata/v3/codec/scale_offset.py | 19 +- .../v3/codec/sharding_indexed.py | 36 ++- .../src/zarr_metadata/v3/codec/transpose.py | 13 +- .../src/zarr_metadata/v3/codec/zstd.py | 19 +- .../zarr_metadata/v3/data_type/_families.py | 19 +- .../src/zarr_metadata/v3/data_type/struct.py | 16 +- .../src/zarr_metadata/v3/entity.py | 46 +-- .../tests/v3/test_acme_affine.py | 36 ++- .../tests/v3/test_acme_decimal.py | 41 ++- .../zarr-metadata/tests/v3/test_entities.py | 16 +- .../tests/v3/test_extension_api.py | 191 +++++++++++-- 21 files changed, 674 insertions(+), 227 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index b41ab7efd2..967f9351a1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -27,17 +27,18 @@ from __future__ import annotations import functools +import operator from abc import ABC, abstractmethod -from collections.abc import Mapping -from dataclasses import dataclass +from collections.abc import Callable, Mapping +from dataclasses import dataclass, is_dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast, get_args -from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._typed_json import ( Loc, Parsed, Parser, + RecordWriter, Writer, as_tuples, declared_class_vars, @@ -48,8 +49,8 @@ parser, parser_for, problem, + record_writer, strip_annotation, - writer, ) if TYPE_CHECKING: @@ -209,40 +210,61 @@ def canonical(self) -> Self: def unreadable(cls: type[MetadataEntity]) -> str | None: """Why `coerce` could not read an instance of `cls`; None if it can. - Three things type-check cleanly and then go wrong somewhere that - will not name the class: a field whose annotation is not a shape - JSON takes, which `coerce` could not parse; a `__post_init__` of the - entity's own, whose rules `coerce` would never ask; and a class - variable a base annotates and nothing sets -- `identifier` for every - entity, `bounds` for an integer type -- which the first lookup would - fail. Registration asks, and refuses the class with the answer. + An entity's fields are `configuration`, a record dataclass of its + members, and at most one field the envelope's name fills. Things + that type-check cleanly and then go wrong somewhere that will not + name the class: a field of any other name; a configuration that is + not a record, or a member of it whose annotation is not a shape JSON + takes; a `__post_init__` of the entity's own, whose rules `coerce` + would never ask; and a class variable a base annotates and nothing + sets -- `identifier` for every entity, `bounds` for an integer type + -- which the first lookup would fail. Registration asks, and refuses + the class with the answer. """ try: hints = field_hints(cls) except NameError as unresolved: - return ( - f"{cls.__name__}: a field annotation names {unresolved.name!r}, which is not " - "defined where the class is; define it at module level, or import it outside " - "`TYPE_CHECKING`" - ) - unread: list[str] = [] + return _unresolved(cls, unresolved) for name, annotation in hints.items(): - try: - accepted = parser_for(annotation, _nested_field) is not None - except TypeError as refused: - return f"{cls.__name__}: {name} {refused}" - if not accepted: - unread.append(name) - if len(unread) != 0: + if name == "configuration" or is_from_name(annotation): + continue return ( - f"{cls.__name__}: " - f"{'; '.join(f'{name} is annotated {hints[name]!r}' for name in unread)}" - ", which is not a shape JSON takes. A field is int, float, bool, str, JSONValue, " - "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " - "record, Mapping[str, V], a NewType, or an entity kind with Opaque " - "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in the function bound as `problems`" + f"{cls.__name__} declares a field {name!r}; an entity's fields are `configuration`, " + "a frozen dataclass of its members, and a name it carries marked FROM_NAME -- put " + f"{name!r} in the configuration record" ) + record = hints.get("configuration") + if record is not None: + if ( + not (isinstance(record, type) and is_dataclass(record)) + or nested_kind(record) is not None + ): + return ( + f"{cls.__name__}: configuration is annotated {record!r}; annotate it with a frozen " + "dataclass of the members, one field per configuration key" + ) + try: + members = field_hints(record) + except NameError as unresolved: + return _unresolved(record, unresolved) + unread: list[str] = [] + for name, annotation in members.items(): + try: + accepted = parser_for(annotation, _nested_field) is not None + except TypeError as refused: + return f"{cls.__name__}: {name} {refused}" + if not accepted: + unread.append(name) + if len(unread) != 0: + return ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {members[name]!r}' for name in unread)}" + ", which is not a shape JSON takes. A member is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in the function bound as `problems`" + ) if "__post_init__" in vars(cls): return ( f"{cls.__name__} defines __post_init__; write its rules as a function of the " @@ -257,6 +279,14 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: return None +def _unresolved(cls: type, unresolved: NameError) -> str: + return ( + f"{cls.__name__}: a field annotation names {unresolved.name!r}, which is not " + "defined where the class is; define it at module level, or import it outside " + "`TYPE_CHECKING`" + ) + + def nested_kind(annotation: object) -> type[MetadataEntity] | None: """The kind an annotation of the form `Kind | Opaque` names; None if it names no entity. @@ -338,35 +368,43 @@ def write(value: object) -> JSONValue: @dataclass(frozen=True, slots=True) -class _Member: - """How `coerce` reads one field: what the annotation says, compiled once.""" +class _Plan: + """How `coerce` reads and `to_json` writes one class, compiled once from its fields.""" - key: str - from_name: bool - optional: bool - parse: Parser[_Reading] - write: Writer + from_name: str | None + """The field the envelope's name fills, for a family; None for every other entity.""" + parse: Parser[_Reading] | None + """The configuration record's parser; None for an entity with no configuration.""" + write: Callable[[MetadataEntity], dict[str, JSONValue]] | None + """The entity's configuration as the JSON object it writes; None for an entity with none.""" + requires_configuration: bool + """Whether the record has a member the document must write.""" @functools.cache -def _plan(cls: type[MetadataEntity]) -> tuple[_Member, ...]: - """The fields of `cls` as `coerce` reads them, compiled once per class. +def _plan(cls: type[MetadataEntity]) -> _Plan: + """The plan for `cls`, a pure function of the class: its fields are fixed once it exists. - A pure function of the class: its fields are fixed once it exists, - and each parser is a function of its annotation alone, taking the - reading it runs in as an argument. `TypeError` for a field no parser - reads, which registration refuses first. + Each parser is a function of its annotation alone, taking the reading + it runs in as an argument. `TypeError` for a shape no parser reads, + which registration refuses first. """ - return tuple( - _Member( - key, - is_from_name(annotation), - is_optional(annotation), - parser(annotation, _nested_field), - writer(annotation, _nested_field_writer), - ) - for key, annotation in field_hints(cls).items() - ) + hints = field_hints(cls) + from_name = next((key for key, annotation in hints.items() if is_from_name(annotation)), None) + record = hints.get("configuration") + if record is None: + return _Plan(from_name, None, None, False) + if not (isinstance(record, type) and is_dataclass(record)): # pragma: no cover - refused first + msg = f"{cls.__name__}: configuration is annotated {record!r}, not a record dataclass" + raise TypeError(msg) + required = any(not is_optional(annotation) for annotation in field_hints(record).values()) + writes: RecordWriter = record_writer(record, _nested_field_writer) + configuration_of = operator.attrgetter("configuration") + + def write(entity: MetadataEntity) -> dict[str, JSONValue]: + return writes(configuration_of(entity)) + + return _Plan(from_name, parser(record, _nested_field), write, required) @dataclass(frozen=True) @@ -452,14 +490,14 @@ def accepts(cls, name: str) -> bool: def coerce(cls, value: object, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one. - Each configuration member is parsed against its field's - annotation; a member holding another entity is read in `context`, - the scope this reading is happening in. An optional member the - document left out is passed as `UNSET`, so no field's default - decides what a document said. The entity is built only when every - member of its own read -- its rules are written over a whole - configuration -- and handed back only when everything inside it - read too. + The configuration is parsed against the record the `configuration` + field names, member by member; a member holding another entity is + read in `context`, the scope this reading is happening in. An + optional member the document left out is `UNSET` in the record, + so no field's default decides what a document said. The entity is + built only when every member of its own read -- its rules are + written over a whole configuration -- and handed back only when + everything inside it read too. """ name, given, envelope = named_configuration(value) if name is None or not cls.accepts(name): @@ -467,60 +505,48 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: if len(envelope) != 0: return None, envelope plan = _plan(cls) - if given is None and any(not member.from_name and not member.optional for member in plan): + members: dict[str, object] = {} + if plan.from_name is not None: + members[plan.from_name] = name + reading = _Reading(context, []) + own: tuple[ValidationProblem, ...] = () + if plan.parse is None: + own = tuple( + found + for key in (given or {}) + for found in problem( + ("configuration", key), f"unexpected key {key!r}", "unknown_key" + ) + ) + elif given is None and plan.requires_configuration: return None, problem( ("configuration",), f"{cls.identifier!r} requires a configuration", "missing_key", ) - configuration: Mapping[str, object] = {} if given is None else given - reading = _Reading(context, []) - declared = {member.key: member for member in plan} - members: dict[str, object] = {} - own: list[ValidationProblem] = [] - for key in configuration: - if key not in declared or declared[key].from_name: - own.extend( - problem(("configuration", key), f"unexpected key {key!r}", "unknown_key") - ) - for member in plan: - key = member.key - if member.from_name: - members[key] = name - elif key not in configuration: - if member.optional: - members[key] = UNSET - else: - own.extend( - problem( - ("configuration", key), f"missing required key {key!r}", "missing_key" - ) - ) - else: - # Arrays as tuples before parsing, so a member holds the - # tuples its type declares, never the lists raw JSON - # arrives as. - members[key], problems = member.parse( - as_tuples(configuration[key]), ("configuration", key), reading - ) - own.extend(problems) + else: + # Arrays as tuples before parsing, so a member holds the + # tuples its type declares, never the lists raw JSON + # arrives as. + members["configuration"], own = plan.parse( + as_tuples({} if given is None else given), ("configuration",), reading + ) found = (*own, *reading.nested) if any(entry.kind != "unknown_key" for entry in own): # An unknown key is survivable; a member that could not be # read is a hole, and judging around it would be guessing. return None, found entity = cls.create_unchecked(**members) - # A problem about a member the envelope's name carries is about - # the entity, and lands on it rather than under a configuration - # the document does not have. - from_name = {member.key for member in plan if member.from_name} + # A problem about the member the envelope's name carries is + # about the entity, and lands on it rather than under a + # configuration the document does not have. refused = within( (), tuple( - ValidationProblem((), found.message, found.kind) - if len(found.loc) != 0 and found.loc[0] in from_name - else found - for found in entity.problems() + ValidationProblem((), entry.message, entry.kind) + if len(entry.loc) != 0 and entry.loc[0] == plan.from_name + else entry + for entry in entity.problems() ), ) if len(refused) != 0: @@ -536,6 +562,19 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: return None, found return entity, found + def with_configuration(self, **changes: object) -> Self: + """This entity with these configuration members changed. + + `codec.with_configuration(typesize=UNSET)` is the record replaced + member by member and the entity rebuilt around it, so the + constructor checks the result as it checks any other. + """ + current = getattr(self, "configuration", None) + if not is_dataclass(current) or isinstance(current, type): + msg = f"{type(self).__name__} has no configuration" + raise TypeError(msg) + return replace(self, configuration=replace(current, **changes)) + def canonical(self) -> Self: """This entity in the simplest form that means the same thing. @@ -556,11 +595,11 @@ def canonical(self) -> Self: def to_json(self) -> ZarrV3MetadataFieldJSON: """This entity as a document would write it. - Written from the fields by the same declaration `coerce` reads - them by, each member by the writer its annotation implies: the - bare name when every member it holds is absent, the object - otherwise, a contained entity through its own `to_json`, a - JSON-valued member copied so the document is not a handle on + Written from the configuration record by the same declaration + `coerce` reads it by, each member by the writer its annotation + implies: the bare name when every member it holds is absent, the + object otherwise, a contained entity through its own `to_json`, + a JSON-valued member copied so the document is not a handle on the entity. Faithful to every member: read a document, write it back, and those come out as they went in. Ask `canonical` first if you want the simplest equivalent spelling. The envelope's @@ -571,14 +610,14 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: An entity whose JSON is not its fields overrides this; none in the package does. """ + plan = _plan(type(self)) name = self.identifier - configuration: dict[str, JSONValue] = {} - for member in _plan(type(self)): - value = getattr(self, member.key) - if member.from_name: - name = value if isinstance(value, str) else name - elif value is not UNSET: - configuration[member.key] = member.write(value) + if plan.from_name is not None: + carried = getattr(self, plan.from_name) + name = carried if isinstance(carried, str) else name + if plan.write is None: + return name + configuration = plan.write(self) if len(configuration) == 0: return name return {"name": name, "configuration": configuration} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index e482e4d0a7..2f3e76abe9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -84,6 +84,9 @@ WriterLeaf: TypeAlias = Callable[[object], "Writer | None"] """A caller's own shapes, for writing: asked first for every annotation, None to decline.""" +RecordWriter: TypeAlias = Callable[[object], dict[str, JSONValue]] +"""A record dataclass as the JSON object a document writes for it.""" + def problem( loc: Loc, message: str, kind: ProblemKind = "invalid_type" @@ -737,10 +740,10 @@ def write(value: object) -> JSONValue: return write -def fields_of(members: Mapping[str, Writer]) -> Writer: +def fields_of(members: Mapping[str, Writer]) -> RecordWriter: """A record dataclass as an object: each field written by its type, an absent optional one left out.""" - def write(value: object) -> JSONValue: + def write(value: object) -> dict[str, JSONValue]: written: dict[str, JSONValue] = {} for key, member in members.items(): entry = getattr(value, key) @@ -833,12 +836,22 @@ def writer(annotation: object, leaf: WriterLeaf) -> Writer: return found +def record_writer(record: type, leaf: WriterLeaf) -> RecordWriter: + """The writer of a record dataclass, as the object it writes; `TypeError` for a field no writer reads.""" + members = _writers_of(field_hints(record), leaf) + if members is None: + msg = f"{record.__name__} has a field that is not a shape JSON takes" + raise TypeError(msg) + return fields_of(members) + + __all__ = [ "Leaf", "Loc", "Members", "Parsed", "Parser", + "RecordWriter", "Writer", "WriterLeaf", "any_of", @@ -868,6 +881,7 @@ def writer(annotation: object, leaf: WriterLeaf) -> Writer: "positions_of", "problem", "record_of", + "record_writer", "sequence_of", "shape_of", "strip_annotation", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index c9a80a49a4..f85645cd2e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -4,7 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast from typing_extensions import TypedDict @@ -166,6 +166,14 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: return frozenset(lengths) if len(lengths) != 0 else None +@dataclass(frozen=True) +class RectilinearChunkGridOptions: + """What a `rectilinear` grid is configured with.""" + + kind: Literal["inline"] + chunk_shapes: tuple[RectilinearDimSpec, ...] + + def rectilinear_problems(grid: "RectilinearChunkGrid", /) -> "Iterator[ValidationProblem]": """Every extent, and every run's length and count, is at least 1.""" for axis, spec in enumerate(grid.chunk_shapes): @@ -187,13 +195,20 @@ def rectilinear_problems(grid: "RectilinearChunkGrid", /) -> "Iterator[Validatio class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" - kind: Literal["inline"] - chunk_shapes: tuple[RectilinearDimSpec, ...] + configuration: RectilinearChunkGridOptions identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME problems = rectilinear_problems + @property + def kind(self) -> Literal["inline"]: + return self.configuration.kind + + @property + def chunk_shapes(self) -> tuple[RectilinearDimSpec, ...]: + return self.configuration.chunk_shapes + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. @@ -241,4 +256,4 @@ def canonical(self) -> Self: Two dimension specs listing the same extents describe the same grid, and the encoded one stays the same size as the array grows. """ - return replace(self, chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) + return self.with_configuration(chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 1310852fb7..e725b5606b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -60,6 +60,13 @@ class RegularChunkGridObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class RegularChunkGridOptions: + """What a `regular` grid is configured with.""" + + chunk_shape: tuple[int, ...] + + def regular_problems(grid: "RegularChunkGrid", /) -> "Iterator[ValidationProblem]": for index, extent in enumerate(grid.chunk_shape): if extent < 1: @@ -72,12 +79,16 @@ def regular_problems(grid: "RegularChunkGrid", /) -> "Iterator[ValidationProblem class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" - chunk_shape: tuple[int, ...] + configuration: RegularChunkGridOptions identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME problems = regular_problems + @property + def chunk_shape(self) -> tuple[int, ...]: + return self.configuration.chunk_shape + def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" if not isinstance(array_shape, (list, tuple)): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 2943d7e6b5..e62f084ef7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -70,10 +70,21 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class DefaultChunkKeyEncodingOptions: + """What the `default` encoding is configured with.""" + + separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET + + @dataclass(frozen=True) class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity): """The `default` chunk key encoding, coerced from its metadata.""" - separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET + configuration: DefaultChunkKeyEncodingOptions identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME + + @property + def separator(self) -> DefaultChunkKeyEncodingSeparator | UNSET: + return self.configuration.separator diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index d41fbbcf1c..3f7d7698d1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -76,10 +76,21 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class V2ChunkKeyEncodingOptions: + """What the `v2` encoding is configured with.""" + + separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET + + @dataclass(frozen=True) class V2ChunkKeyEncoding(ChunkKeyEncodingEntity): """The `v2` chunk key encoding, coerced from its metadata.""" - separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET + configuration: V2ChunkKeyEncodingOptions identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME + + @property + def separator(self) -> V2ChunkKeyEncodingSeparator | UNSET: + return self.configuration.separator diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 7d1096faf5..ca6e57c65d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -4,7 +4,7 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html """ -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict @@ -89,6 +89,17 @@ class BloscCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class BloscOptions: + """What `blosc` is configured with.""" + + cname: BloscCName + clevel: int + shuffle: BloscShuffle + blocksize: int + typesize: int | UNSET = UNSET + + def blosc_problems(codec: "BloscCodec", /) -> "Iterator[ValidationProblem]": """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. @@ -128,11 +139,7 @@ class BloscCodec(BytesBytesCodec): equivalent document. """ - cname: BloscCName - clevel: int - shuffle: BloscShuffle - blocksize: int - typesize: int | UNSET = UNSET + configuration: BloscOptions identifier: ClassVar[str] = BLOSC_CODEC_NAME variable_size: ClassVar[bool] = True @@ -141,6 +148,26 @@ class BloscCodec(BytesBytesCodec): # when shuffling; `blosc_problems` is where that conditional lives. problems = blosc_problems + @property + def cname(self) -> BloscCName: + return self.configuration.cname + + @property + def clevel(self) -> int: + return self.configuration.clevel + + @property + def shuffle(self) -> BloscShuffle: + return self.configuration.shuffle + + @property + def blocksize(self) -> int: + return self.configuration.blocksize + + @property + def typesize(self) -> int | UNSET: + return self.configuration.typesize + def canonical(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. @@ -149,4 +176,4 @@ def canonical(self) -> Self: """ if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: return self - return replace(self, typesize=UNSET) + return self.with_configuration(typesize=UNSET) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index c1753db981..ace8a64a17 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -78,6 +78,13 @@ class BytesCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class BytesOptions: + """What `bytes` is configured with.""" + + endian: Endianness | UNSET = UNSET + + @dataclass(frozen=True) class BytesCodec(ArrayBytesCodec): """The `bytes` codec, coerced from its metadata. @@ -86,11 +93,15 @@ class BytesCodec(ArrayBytesCodec): has no byte order to state, and the spec lets such an array omit it. """ - endian: Endianness | UNSET = UNSET + configuration: BytesOptions identifier: ClassVar[str] = BYTES_CODEC_NAME variable_size: ClassVar[bool] = False + @property + def endian(self) -> Endianness | UNSET: + return self.configuration.endian + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """The data type reaching here must have a raw byte representation. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index fd65f622ce..313fbfecc5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -4,7 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict @@ -124,6 +124,16 @@ class CastValueCodecObject(TypedDict, closed=True): """The two directions a `scalar_map` can override, both optional.""" +@dataclass(frozen=True) +class CastValueOptions: + """What `cast_value` is configured with.""" + + data_type: DataTypeEntity | Opaque + rounding: CastRoundingMode | UNSET = UNSET + out_of_range: CastOutOfRangeMode | UNSET = UNSET + scalar_map: ScalarMap | UNSET = UNSET + + @dataclass(frozen=True) class CastValueCodec(ArrayArrayCodec): """The `cast_value` codec, coerced from its metadata. @@ -132,17 +142,30 @@ class CastValueCodec(ArrayArrayCodec): read in a scope rather than on its own. """ - data_type: DataTypeEntity | Opaque - rounding: CastRoundingMode | UNSET = UNSET - out_of_range: CastOutOfRangeMode | UNSET = UNSET - scalar_map: ScalarMap | UNSET = UNSET + configuration: CastValueOptions identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME variable_size: ClassVar[bool] = False + @property + def data_type(self) -> DataTypeEntity | Opaque: + return self.configuration.data_type + + @property + def rounding(self) -> CastRoundingMode | UNSET: + return self.configuration.rounding + + @property + def out_of_range(self) -> CastOutOfRangeMode | UNSET: + return self.configuration.out_of_range + + @property + def scalar_map(self) -> ScalarMap | UNSET: + return self.configuration.scalar_map + def canonical(self) -> Self: """The target data type in its own canonical form.""" - return replace(self, data_type=self.data_type.canonical()) + return self.with_configuration(data_type=self.data_type.canonical()) def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index ec17ba640e..3cc333c00b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -67,6 +67,13 @@ class GzipCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class GzipOptions: + """What `gzip` is configured with.""" + + level: int + + def gzip_problems(codec: "GzipCodec", /) -> "Iterator[ValidationProblem]": if not 0 <= codec.level <= 9: yield ValidationProblem( @@ -78,9 +85,13 @@ def gzip_problems(codec: "GzipCodec", /) -> "Iterator[ValidationProblem]": class GzipCodec(BytesBytesCodec): """The `gzip` codec, coerced from its metadata.""" - level: int + configuration: GzipOptions identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True problems = gzip_problems + + @property + def level(self) -> int: + return self.configuration.level diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index f51fd13aa0..a1fb70f459 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -74,6 +74,14 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class ScaleOffsetOptions: + """What `scale_offset` is configured with.""" + + offset: JSONValue | UNSET = UNSET + scale: JSONValue | UNSET = UNSET + + def scale_offset_problems(codec: "ScaleOffsetCodec", /) -> "Iterator[ValidationProblem]": """Each value is a scalar of the array's type, so neither is null. @@ -97,14 +105,21 @@ class ScaleOffsetCodec(ArrayArrayCodec): is a question for the rules layer. """ - offset: JSONValue | UNSET = UNSET - scale: JSONValue | UNSET = UNSET + configuration: ScaleOffsetOptions identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME variable_size: ClassVar[bool] = False problems = scale_offset_problems + @property + def offset(self) -> JSONValue | UNSET: + return self.configuration.offset + + @property + def scale(self) -> JSONValue | UNSET: + return self.configuration.scale + def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same array, element for element. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 02634c2f14..f5aa1b701e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -4,7 +4,7 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html """ -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict @@ -97,6 +97,16 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class ShardingIndexedOptions: + """What `sharding_indexed` is configured with.""" + + chunk_shape: tuple[int, ...] + codecs: tuple[CodecEntity | Opaque, ...] + index_codecs: tuple[CodecEntity | Opaque, ...] + index_location: ShardingIndexLocation | UNSET = UNSET + + def sharding_problems(codec: "ShardingIndexedCodec", /) -> "Iterator[ValidationProblem]": for index, extent in enumerate(codec.chunk_shape): if extent < 1: @@ -114,20 +124,32 @@ class ShardingIndexedCodec(ArrayBytesCodec): itself an entity, read the same way this one was. """ - chunk_shape: tuple[int, ...] - codecs: tuple[CodecEntity | Opaque, ...] - index_codecs: tuple[CodecEntity | Opaque, ...] - index_location: ShardingIndexLocation | UNSET = UNSET + configuration: ShardingIndexedOptions identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME variable_size: ClassVar[bool] = True problems = sharding_problems + @property + def chunk_shape(self) -> tuple[int, ...]: + return self.configuration.chunk_shape + + @property + def codecs(self) -> tuple[CodecEntity | Opaque, ...]: + return self.configuration.codecs + + @property + def index_codecs(self) -> tuple[CodecEntity | Opaque, ...]: + return self.configuration.index_codecs + + @property + def index_location(self) -> ShardingIndexLocation | UNSET: + return self.configuration.index_location + def canonical(self) -> Self: """Each pipeline's codecs in their own canonical form.""" - return replace( - self, + return self.with_configuration( codecs=tuple(codec.canonical() for codec in self.codecs), index_codecs=tuple(codec.canonical() for codec in self.index_codecs), ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index cbf7af3c62..a1f8160f3d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -64,6 +64,13 @@ class TransposeCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class TransposeOptions: + """What `transpose` is configured with.""" + + order: tuple[int, ...] + + def transpose_problems(codec: "TransposeCodec", /) -> "Iterator[ValidationProblem]": """`order` must permute its own axes. @@ -82,13 +89,17 @@ def transpose_problems(codec: "TransposeCodec", /) -> "Iterator[ValidationProble class TransposeCodec(ArrayArrayCodec): """The `transpose` codec, coerced from its metadata.""" - order: tuple[int, ...] + configuration: TransposeOptions identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME variable_size: ClassVar[bool] = False problems = transpose_problems + @property + def order(self) -> tuple[int, ...]: + return self.configuration.order + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 63f7d25d39..4b802df965 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -75,6 +75,14 @@ class ZstdCodecObject(TypedDict, closed=True): ] +@dataclass(frozen=True) +class ZstdOptions: + """What `zstd` is configured with.""" + + level: int + checksum: bool | UNSET = UNSET + + def zstd_problems(codec: "ZstdCodec", /) -> "Iterator[ValidationProblem]": if not ZSTD_MIN_LEVEL <= codec.level <= ZSTD_MAX_LEVEL: yield ValidationProblem( @@ -88,10 +96,17 @@ def zstd_problems(codec: "ZstdCodec", /) -> "Iterator[ValidationProblem]": class ZstdCodec(BytesBytesCodec): """The `zstd` codec, coerced from its metadata.""" - level: int - checksum: bool | UNSET = UNSET + configuration: ZstdOptions identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True problems = zstd_problems + + @property + def level(self) -> int: + return self.configuration.level + + @property + def checksum(self) -> bool | UNSET: + return self.configuration.checksum diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 01115045cc..e9766aec66 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -167,6 +167,14 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP """The largest `scale_factor` numpy stores: the field is a signed int32.""" +@dataclass(frozen=True) +class NumpyTimeOptions: + """What a numpy time type is configured with: a unit, and how many of it one tick is.""" + + unit: NumpyTimeUnit + scale_factor: int + + def numpy_time_problems(data_type: NumpyTimeDataType, /) -> Iterator[ValidationProblem]: if not 1 <= data_type.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: yield ValidationProblem( @@ -186,12 +194,19 @@ class NumpyTimeDataType(DataTypeEntity): neither sibling imports them from the other. """ - unit: NumpyTimeUnit - scale_factor: int + configuration: NumpyTimeOptions scalar_storage: ClassVar[StorageClass] = "multi_byte" problems = numpy_time_problems + @property + def unit(self) -> NumpyTimeUnit: + return self.configuration.unit + + @property + def scale_factor(self) -> int: + return self.configuration.scale_factor + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: if value == "NaT": return () diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 852dec071e..61502d61b0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -98,6 +98,13 @@ class StructFieldComponent: data_type: DataTypeEntity | Opaque +@dataclass(frozen=True) +class StructOptions: + """What `struct` is configured with.""" + + fields: tuple[StructFieldComponent, ...] + + def struct_problems(data_type: "StructDataType", /) -> "Iterator[ValidationProblem]": """Names exist, are non-empty and distinct; types are fixed-size. @@ -140,17 +147,20 @@ class StructDataType(DataTypeEntity): read in to make sense of them. """ - fields: tuple[StructFieldComponent, ...] + configuration: StructOptions identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" problems = struct_problems + @property + def fields(self) -> tuple[StructFieldComponent, ...]: + return self.configuration.fields + def canonical(self) -> Self: """Each field's data type in its own canonical form.""" - return replace( - self, + return self.with_configuration( fields=tuple( replace(field, data_type=field.data_type.canonical()) for field in self.fields ), diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 82784f3fd0..e3e4e0a464 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -39,8 +39,9 @@ **Writing an extension.** Subclass the kind of thing it is -- a codec's kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity` or -`StorageTransformerEntity`; declare the configuration as dataclass -fields; write every rule finer than a type as a function of the +`StorageTransformerEntity`; declare the configuration as a frozen +dataclass of its members and name it in the entity's one field, +`configuration`; write every rule finer than a type as a function of the instance that yields problems, and bind it as `problems`; add the class to a scope. Complete; runnable given a `document`: @@ -56,17 +57,22 @@ ValidationProblem, ) + @dataclass(frozen=True) # the fields are the schema; frozen, so a configuration is a value + class AcmeLz4Options: + acceleration: int | UNSET = UNSET # optional: absent reads as UNSET + def acme_lz4_problems(codec: "AcmeLz4Codec", /) -> Iterator[ValidationProblem]: - if codec.acceleration is not UNSET and not 1 <= codec.acceleration <= 65537: + acceleration = codec.configuration.acceleration + if acceleration is not UNSET and not 1 <= acceleration <= 65537: yield ValidationProblem( ("acceleration",), - f"expected an integer in [1, 65537], got {codec.acceleration}", + f"expected an integer in [1, 65537], got {acceleration}", "invalid_value", ) - @dataclass(frozen=True) # the fields are the schema; frozen, so an entity is a value + @dataclass(frozen=True) class AcmeLz4Codec(BytesBytesCodec): - acceleration: int | UNSET = UNSET # optional: absent reads as UNSET + configuration: AcmeLz4Options # the shape of the metadata: a name, and a configuration identifier: ClassVar[str] = "acme.lz4" variable_size: ClassVar[bool] = True # a compressor: its output length is not fixed @@ -75,10 +81,12 @@ class AcmeLz4Codec(BytesBytesCodec): SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec) validate_array_metadata_v3(document, context=SCOPE) -The fields are the only place the shape is written. Which members exist, -which may be left out (the type admits `UNSET`), how each one is -type-checked, and how each is written back are all read off the -annotations, and the shapes are the +An entity has the shape of its metadata: a name, which is the class, +and a configuration, which is a record dataclass named in the one field +`configuration`. The record's fields are the only place the members +are written. Which members exist, which may be left out (the type +admits `UNSET`), how each one is type-checked, and how each is written +back are all read off the annotations, and the shapes are the ones JSON takes: `int`, `float` (any JSON number), `bool`, `str`, `JSONValue`, a `Literal` of names, `tuple[T, ...]` or `tuple[T1, T2]`, a TypedDict or dataclass record, `Mapping[str, V]`, a `NewType`, and a @@ -89,7 +97,11 @@ class AcmeLz4Codec(BytesBytesCodec): registration. A required member has no default; an optional one is `| UNSET = UNSET`, so absence stays distinct from a JSON `null`, and a member that means something when -absent is read that way where it is used, not defaulted. +absent is read that way where it is used, not defaulted. A member is +read as `codec.configuration.acceleration`; an entity that wants it at +the top level adds a `@property` for it. `with_configuration(**changes)` +is the entity with members of its configuration replaced, checked as +any construction is. Everything finer than a type -- a bound, a rule about one member, members read together -- is a function of the instance that yields @@ -104,14 +116,14 @@ class AcmeLz4Codec(BytesBytesCodec): It runs only on an entity whose members all read: a member of the wrong type is reported and the entity is not built. -**What an entity answers for itself**, beyond its fields. `to_json` is -written once in the base, from the fields: the bare name when every +**What an entity answers for itself**, beyond its configuration. `to_json` is +written once in the base, from the record: the bare name when every member is absent, the object otherwise, a contained entity through its own `to_json`; an entity whose JSON is not its fields overrides it, and none in the package does. `canonical`, the entity in its simplest equivalent form: the entity itself by default, overridden where two spellings of its members mean the same, and in an entity that contains entities to put -those in canonical form -- `replace(self, inner=self.inner.canonical())`. +those in canonical form -- `self.with_configuration(inner=self.inner.canonical())`. An `Opaque` answers both as well, with the JSON it kept and with itself, so a field typed `CodecEntity | Opaque` is written and simplified without asking which it holds. `coerce` is written once in the base. Then, by @@ -141,8 +153,10 @@ class AcmeLz4Codec(BytesBytesCodec): Registration is the one moment an entity is refused, with a message that says what to write: a class without `@dataclass`, a codec -subclassing `CodecEntity` instead of a kind, a field whose annotation is -not a shape JSON takes -- a nested entity without `Opaque` among them -- +subclassing `CodecEntity` instead of a kind, a field other than +`configuration` and a carried name, a configuration that is not a +record dataclass, a member whose annotation is not a shape JSON takes +-- a nested entity without `Opaque` among them -- a `__post_init__` of the entity's own, a class variable a base annotates and nothing sets, and what a kind leaves abstract. Everything else an author could get wrong, pyright says in the editor: the fields, diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index bbbd1813a6..99e1eb4e21 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -8,7 +8,7 @@ from __future__ import annotations -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Literal, NotRequired, Self import pytest @@ -62,21 +62,37 @@ def acme_affine_problems(codec: AcmeAffineCodec, /) -> Iterator[ValidationProble @dataclass(frozen=True) -class AcmeAffineCodec(ArrayArrayCodec): - """`x * scale + offset`, stored as `dtype` if one is named.""" - +class AcmeAffineOptions: scale: float offset: float | UNSET = UNSET dtype: DataTypeEntity | Opaque | UNSET = UNSET + +@dataclass(frozen=True) +class AcmeAffineCodec(ArrayArrayCodec): + """`x * scale + offset`, stored as `dtype` if one is named.""" + + configuration: AcmeAffineOptions + identifier: ClassVar[str] = "acme.affine" variable_size: ClassVar[bool] = False problems = acme_affine_problems + @property + def scale(self) -> float: + return self.configuration.scale + + @property + def offset(self) -> float | UNSET: + return self.configuration.offset + + @property + def dtype(self) -> DataTypeEntity | Opaque | UNSET: + return self.configuration.dtype + def canonical(self) -> Self: """An offset of 0 is the identity, and absent says the same; `dtype` in its own form.""" - return replace( - self, + return self.with_configuration( offset=UNSET if self.offset == 0 else self.offset, dtype=UNSET if self.dtype is UNSET else self.dtype.canonical(), ) @@ -197,7 +213,9 @@ def test_round_trip_and_canonical() -> None: codec = ArrayDocumentV3.from_json(_document(codecs=[entry, BYTES_LE]), context=SCOPE).codecs[0] assert isinstance(codec, AcmeAffineCodec) assert codec.to_json() == entry - assert AcmeAffineCodec(scale=2, offset=0).canonical() == AcmeAffineCodec(scale=2) + assert AcmeAffineCodec(AcmeAffineOptions(scale=2, offset=0)).canonical() == AcmeAffineCodec( + AcmeAffineOptions(scale=2) + ) document = _document(codecs=[_affine(scale=2, offset=0.0), BYTES_LE]) result = canonicalize_array_metadata_v3(document, context=SCOPE) assert isinstance(result, Canonical) @@ -205,13 +223,13 @@ def test_round_trip_and_canonical() -> None: def test_constructed_by_hand() -> None: - codec = AcmeAffineCodec(scale=2.5, offset=-1, dtype=Float32DataType()) + codec = AcmeAffineCodec(AcmeAffineOptions(scale=2.5, offset=-1, dtype=Float32DataType())) assert codec.to_json() == { "name": "acme.affine", "configuration": {"scale": 2.5, "offset": -1, "dtype": "float32"}, } with pytest.raises(MetadataValidationError) as caught: - AcmeAffineCodec(scale=0) + AcmeAffineCodec(AcmeAffineOptions(scale=0)) assert [(found.loc, found.kind) for found in caught.value.problems] == [ (("scale",), "invalid_value") ] diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 32374cfa0a..8a658448e3 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -89,17 +89,30 @@ def acme_decimal_problems(data_type: AcmeDecimalDataType, /) -> Iterator[Validat ) +@dataclass(frozen=True) +class AcmeDecimalOptions: + precision: int + scale: int + + @dataclass(frozen=True) class AcmeDecimalDataType(DataTypeEntity): """The `acme.decimal` data type, coerced from its metadata.""" - precision: int - scale: int + configuration: AcmeDecimalOptions identifier: ClassVar[str] = ACME_DECIMAL_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "multi_byte" problems = acme_decimal_problems + @property + def precision(self) -> int: + return self.configuration.precision + + @property + def scale(self) -> int: + return self.configuration.scale + def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """A decimal literal whose digits fit `precision` and `scale`. @@ -261,11 +274,11 @@ def test_canonical_form_keeps_the_configuration() -> None: def test_hand_construction() -> None: - entity = AcmeDecimalDataType(precision=4, scale=2) + entity = AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=2)) assert entity.storage_class() == "multi_byte" assert entity.to_json() == _data_type(4, 2) - assert entity == AcmeDecimalDataType(4, 2) - assert hash(entity) == hash(AcmeDecimalDataType(4, 2)) + assert entity == AcmeDecimalDataType(AcmeDecimalOptions(4, 2)) + assert hash(entity) == hash(AcmeDecimalDataType(AcmeDecimalOptions(4, 2))) @pytest.mark.parametrize( @@ -283,26 +296,26 @@ def test_hand_construction() -> None: ], ) def test_fill_values_that_fit_are_accepted(precision: int, scale: int, value: str) -> None: - entity = AcmeDecimalDataType(precision=precision, scale=scale) + entity = AcmeDecimalDataType(AcmeDecimalOptions(precision=precision, scale=scale)) assert entity.fill_value_problems(value, ("fill_value",)) == () @pytest.mark.parametrize("precision", [0, 39]) def test_error_precision_out_of_range(precision: int) -> None: with pytest.raises(MetadataValidationError) as caught: - AcmeDecimalDataType(precision=precision, scale=0) + AcmeDecimalDataType(AcmeDecimalOptions(precision=precision, scale=0)) assert _locs(caught.value.problems) == [("precision",)] def test_error_scale_negative() -> None: with pytest.raises(MetadataValidationError) as caught: - AcmeDecimalDataType(precision=4, scale=-1) + AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=-1)) assert _locs(caught.value.problems) == [("scale",)] def test_error_scale_above_precision() -> None: with pytest.raises(MetadataValidationError) as caught: - AcmeDecimalDataType(precision=4, scale=5) + AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=5)) assert [(problem.loc, problem.message) for problem in caught.value.problems] == [ (("scale",), "expected an integer <= precision (4), got 5") ] @@ -310,7 +323,7 @@ def test_error_scale_above_precision() -> None: def test_error_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: with pytest.raises(MetadataValidationError) as caught: - AcmeDecimalDataType(precision=0, scale=-1) + AcmeDecimalDataType(AcmeDecimalOptions(precision=0, scale=-1)) assert _locs(caught.value.problems) == [("precision",)] _, problems = SCOPE.coerce( DataTypeEntity, {"name": "acme.decimal", "configuration": {"precision": 0, "scale": -1}} @@ -319,18 +332,18 @@ def test_error_the_constructor_stops_at_the_first_problem_and_coerce_reports_eve def test_error_fill_value_must_be_a_string() -> None: - entity = AcmeDecimalDataType(precision=4, scale=2) + entity = AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=2)) assert _locs(entity.fill_value_problems(12.5, ("fill_value",))) == [("fill_value",)] @pytest.mark.parametrize("value", ["", "1e2", " 12.5", "12.", ".5", "abc", "1,5", "NaN"]) def test_error_fill_value_must_be_a_decimal_literal(value: str) -> None: - entity = AcmeDecimalDataType(precision=4, scale=2) + entity = AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=2)) assert _locs(entity.fill_value_problems(value, ("fill_value",))) == [("fill_value",)] def test_error_fill_value_with_too_many_fraction_digits() -> None: - entity = AcmeDecimalDataType(precision=4, scale=2) + entity = AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=2)) problems = entity.fill_value_problems("1.234", ("fill_value",)) assert [(problem.loc, problem.message) for problem in problems] == [ (("fill_value",), "'1.234' has 3 fractional digits, but scale is 2") @@ -338,7 +351,7 @@ def test_error_fill_value_with_too_many_fraction_digits() -> None: def test_error_fill_value_with_too_many_integer_digits() -> None: - entity = AcmeDecimalDataType(precision=4, scale=2) + entity = AcmeDecimalDataType(AcmeDecimalOptions(precision=4, scale=2)) problems = entity.fill_value_problems("1234.5", ("fill_value",)) assert [(problem.loc, problem.message) for problem in problems] == [ ( diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index c8a0c41546..95a3ab54b7 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -736,16 +736,25 @@ def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: # A storage transformer: the one extension point nothing in the package # models, so the only way to reach it is to register one. +@dataclasses.dataclass(frozen=True) +class AcmeShardCacheOptions: + verbose: bool | UNSET = UNSET + + @dataclasses.dataclass(frozen=True) class AcmeShardCache(StorageTransformerEntity): """A third-party storage transformer with a member canonical form drops.""" - verbose: bool | UNSET = UNSET + configuration: AcmeShardCacheOptions identifier: ClassVar[str] = "acme.shard_cache" + @property + def verbose(self) -> bool | UNSET: + return self.configuration.verbose + def canonical(self) -> Self: - return dataclasses.replace(self, verbose=UNSET) + return self.with_configuration(verbose=UNSET) def test_the_document_writes_itself_back_and_canonical_reaches_every_point() -> None: @@ -797,7 +806,8 @@ def test_the_fields_are_the_public_configuration_type() -> None: # writes. Nothing else ties the two, so this does: same keys, same # requiredness. An entity of no members has no such type. for cls in CORE_AND_EXTENSIONS.entities(): - hints = field_hints(cls) + record = field_hints(cls).get("configuration") + hints = field_hints(record) if isinstance(record, type) else {} members = {key for key, annotation in hints.items() if not is_from_name(annotation)} required = {key for key in members if not is_optional(hints[key])} declared = [ diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index f77440b7fd..3bd3e23d84 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self import pytest @@ -18,7 +18,7 @@ canonicalize_array_metadata_v3, validate_array_metadata_v3, ) -from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.codec.blosc import BloscCodec, BloscOptions from zarr_metadata.v3.codec.gzip import GzipCodec from zarr_metadata.v3.entity import ( CORE_AND_EXTENSIONS, @@ -56,16 +56,25 @@ def acme_lz4_problems(codec: AcmeLz4Codec, /) -> Iterator[ValidationProblem]: ) +@dataclass(frozen=True) +class AcmeLz4Options: + acceleration: int | UNSET = UNSET + + @dataclass(frozen=True) class AcmeLz4Codec(BytesBytesCodec): """A third-party compressor.""" - acceleration: int | UNSET = UNSET + configuration: AcmeLz4Options identifier: ClassVar[str] = "acme.lz4" variable_size: ClassVar[bool] = True problems = acme_lz4_problems + @property + def acceleration(self) -> int | UNSET: + return self.configuration.acceleration + @dataclass(frozen=True) class AcmeFloat8DataType(DataTypeEntity): @@ -177,18 +186,28 @@ def test_the_entity_layer_answers_what_a_reader_needs() -> None: assert parts.grid.axis(0) == frozenset({32}) +@dataclass(frozen=True) +class DefaultedOptions: + level: int | UNSET = 3 + + def test_an_absent_optional_member_is_read_as_unset_whatever_its_default() -> None: # A default is for hand construction; what a document left out is - # `UNSET`, so no field's default decides what a document said. + # `UNSET` in the record, so no field's default decides what a + # document said. @dataclass(frozen=True) class Defaulted(BytesBytesCodec): - level: int | UNSET = 3 + configuration: DefaultedOptions identifier: ClassVar[str] = "acme.defaulted" variable_size: ClassVar[bool] = False - assert Defaulted().level == 3 + @property + def level(self) -> int | UNSET: + return self.configuration.level + + assert Defaulted(DefaultedOptions()).level == 3 codec, problems = CORE_AND_EXTENSIONS.extended_with(Defaulted).coerce( CodecEntity, "acme.defaulted" ) @@ -325,34 +344,52 @@ def test_a_third_party_can_register_a_family() -> None: assert [(p.loc, p.kind) for p in problems] == [(("data_type",), "invalid_value")] +@dataclass(frozen=True) +class StructuredOptions: + inner: object + + def test_error_a_member_needs_a_check_from_somewhere() -> None: # An annotation outside the shapes the parser reads implies no # parser, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. @dataclass(frozen=True) class Structured(BytesBytesCodec): - inner: object + configuration: StructuredOptions identifier: ClassVar[str] = "acme.structured" variable_size: ClassVar[bool] = False + @property + def inner(self) -> object: + return self.configuration.inner + with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): CORE_AND_EXTENSIONS.extended_with(Structured) # A third-party codec that contains another codec. +@dataclass(frozen=True) +class AcmeWrapperOptions: + inner: CodecEntity | Opaque + + @dataclass(frozen=True) class AcmeWrapperCodec(BytesBytesCodec): """A codec that applies another codec after its own step.""" - inner: CodecEntity | Opaque + configuration: AcmeWrapperOptions identifier: ClassVar[str] = "acme.wrapper" variable_size: ClassVar[bool] = False + @property + def inner(self) -> CodecEntity | Opaque: + return self.configuration.inner + def canonical(self) -> Self: - return replace(self, inner=self.inner.canonical()) + return self.with_configuration(inner=self.inner.canonical()) def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: @@ -413,6 +450,12 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: assert inner.typesize is UNSET +@dataclass(frozen=True) +class AcmeFramedOptions: + inner: CodecEntity | Opaque + frame: int | UNSET = UNSET + + def test_canonical_is_the_entity_s_own_and_reaches_what_it_contains() -> None: # An entity that contains an entity and rewrites its own members does # both in one `canonical` -- the contained blosc loses the `typesize` @@ -420,36 +463,55 @@ def test_canonical_is_the_entity_s_own_and_reaches_what_it_contains() -> None: # is dropped -- with nothing to call `super()` for. @dataclass(frozen=True) class AcmeFramedCodec(BytesBytesCodec): - inner: CodecEntity | Opaque - frame: int | UNSET = UNSET + configuration: AcmeFramedOptions identifier: ClassVar[str] = "acme.framed" variable_size: ClassVar[bool] = False + @property + def inner(self) -> CodecEntity | Opaque: + return self.configuration.inner + + @property + def frame(self) -> int | UNSET: + return self.configuration.frame + def canonical(self) -> Self: - return replace( - self, + return self.with_configuration( inner=self.inner.canonical(), frame=UNSET if self.frame == 0 else self.frame, ) - blosc = BloscCodec(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) - framed = AcmeFramedCodec(inner=blosc, frame=0) - assert framed.canonical() == AcmeFramedCodec(inner=replace(blosc, typesize=UNSET)) + blosc = BloscCodec( + BloscOptions(cname="zstd", clevel=5, shuffle="noshuffle", typesize=4, blocksize=0) + ) + framed = AcmeFramedCodec(AcmeFramedOptions(inner=blosc, frame=0)) + assert framed.canonical() == AcmeFramedCodec( + AcmeFramedOptions(inner=blosc.with_configuration(typesize=UNSET)) + ) assert framed.inner is blosc # a transformation, not a mutation +@dataclass(frozen=True) +class VagueOptions: + inner: MetadataEntity | Opaque + + def test_error_a_nested_field_names_a_kind() -> None: # `MetadataEntity` is of no kind, so a field typed as one could not # be resolved through any scope. @dataclass(frozen=True) class Vague(BytesBytesCodec): - inner: MetadataEntity | Opaque + configuration: VagueOptions identifier: ClassVar[str] = "acme.vague" variable_size: ClassVar[bool] = False + @property + def inner(self) -> MetadataEntity | Opaque: + return self.configuration.inner + with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): CORE_AND_EXTENSIONS.extended_with(Vague) @@ -484,17 +546,26 @@ def acme_block_problems(codec: AcmeBlockCodec, /) -> Iterator[ValidationProblem] ) +@dataclass(frozen=True) +class AcmeBlockOptions: + block: int + + @dataclass(frozen=True) class AcmeBlockCodec(BytesBytesCodec): """A codec whose block size must be a power of two.""" - block: int + configuration: AcmeBlockOptions identifier: ClassVar[str] = "acme.block" variable_size: ClassVar[bool] = False problems = acme_block_problems + @property + def block(self) -> int: + return self.configuration.block + def test_a_rule_about_a_member_is_a_function_of_the_instance() -> None: # The rule runs on the typed members and reports relative to the @@ -512,7 +583,7 @@ def test_a_rule_about_a_member_is_a_function_of_the_instance() -> None: ] # And on the constructor, the same rule. with pytest.raises(MetadataValidationError) as caught: - AcmeBlockCodec(block=6) + AcmeBlockCodec(AcmeBlockOptions(block=6)) assert [p.loc for p in caught.value.problems] == [("block",)] # A member that failed its type check never reaches the rule. _, problems = scope.coerce(CodecEntity, {"name": "acme.block", "configuration": {"block": "x"}}) @@ -530,35 +601,48 @@ def acme_range_problems(codec: AcmeRangeCodec, /) -> Iterator[ValidationProblem] ) +@dataclass(frozen=True) +class AcmeRangeOptions: + low: int + high: int + + @dataclass(frozen=True) class AcmeRangeCodec(BytesBytesCodec): """A codec with two rules, so that one can fail after another.""" - low: int - high: int + configuration: AcmeRangeOptions identifier: ClassVar[str] = "acme.range" variable_size: ClassVar[bool] = False problems = acme_range_problems + @property + def low(self) -> int: + return self.configuration.low + + @property + def high(self) -> int: + return self.configuration.high + def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: # One function, two consumers: the constructor takes the first # problem it yields, `coerce` runs it to the end. A consumer holding # an entity may run it too, and stop or collect as it likes. with pytest.raises(MetadataValidationError) as caught: - AcmeRangeCodec(low=-1, high=-2) + AcmeRangeCodec(AcmeRangeOptions(low=-1, high=-2)) assert [p.loc for p in caught.value.problems] == [("low",)] scope = CORE_AND_EXTENSIONS.extended_with(AcmeRangeCodec) _, problems = scope.coerce( CodecEntity, {"name": "acme.range", "configuration": {"low": -1, "high": -2}} ) assert [p.loc for p in problems] == [("configuration", "low"), ("configuration", "high")] - assert list(acme_range_problems(AcmeRangeCodec(low=0, high=1))) == [] + assert list(acme_range_problems(AcmeRangeCodec(AcmeRangeOptions(low=0, high=1)))) == [] # A reader that wants every problem of a hand-built one builds the # record without the check and asks. - record = AcmeRangeCodec.create_unchecked(low=-1, high=-2) + record = AcmeRangeCodec.create_unchecked(configuration=AcmeRangeOptions(low=-1, high=-2)) assert [p.loc for p in record.problems()] == [("low",), ("high",)] @@ -577,6 +661,13 @@ def __post_init__(self) -> None: CORE_AND_EXTENSIONS.extended_with(Checked) +@dataclass(frozen=True) +class LocalizedOptions: + # `Local` is defined inside the test, so it is not here, where this + # class's annotations resolve: the case the message is for. + inner: Local # noqa: F821 # pyright: ignore[reportUndefinedVariable] + + def test_error_a_field_annotation_names_what_is_not_defined() -> None: # Annotations are resolved where the class is, at registration; a # type defined inside a function is not there. @@ -585,11 +676,15 @@ class Local(TypedDict, closed=True): @dataclass(frozen=True) class Localized(BytesBytesCodec): - inner: Local + configuration: LocalizedOptions identifier: ClassVar[str] = "acme.localized" variable_size: ClassVar[bool] = False + @property + def inner(self) -> Local: + return self.configuration.inner + with pytest.raises(TypeError, match="a field annotation names 'Local', which is not defined"): CORE_AND_EXTENSIONS.extended_with(Localized) @@ -622,18 +717,27 @@ def test_a_malformed_envelope_is_one_problem() -> None: assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "invalid_type")] +@dataclass(frozen=True) +class AcmeSlottedOptions: + level: int + + def test_a_slotted_entity_is_accepted() -> None: # `@dataclass(slots=True)` builds the class twice; registration sees # the second, whose members are slot descriptors. @dataclass(frozen=True, slots=True) class AcmeSlotted(BytesBytesCodec): - level: int + configuration: AcmeSlottedOptions identifier: ClassVar[str] = "acme.slotted" variable_size: ClassVar[bool] = False - assert AcmeSlotted(level=1).to_json() == { + @property + def level(self) -> int: + return self.configuration.level + + assert AcmeSlotted(AcmeSlottedOptions(level=1)).to_json() == { "name": "acme.slotted", "configuration": {"level": 1}, } @@ -649,18 +753,27 @@ class AcmeNoted(BytesBytesCodec): assert AcmeNoted().to_json() == "acme.noted" +@dataclass(frozen=True) +class AcmeScaledOptions: + scale: float + + def test_a_number_member_is_a_float_field() -> None: # JSON has one number type; `float` admits an int spelled without a # point and refuses a bool, which is what a document's `2` and `true` # deserve. @dataclass(frozen=True) class AcmeScaled(ArrayArrayCodec): - scale: float + configuration: AcmeScaledOptions identifier: ClassVar[str] = "acme.scaled" variable_size: ClassVar[bool] = False + @property + def scale(self) -> float: + return self.configuration.scale + def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming @@ -679,28 +792,46 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: ] +@dataclass(frozen=True) +class UndecoratedOptions: + level: int + + def test_error_an_entity_must_be_a_dataclass() -> None: # Class creation runs before `@dataclass` and cannot see it missing; # registration can, and says so instead of the first `coerce` failing # with the base class's `__init__`. class Undecorated(BytesBytesCodec): - level: int + configuration: UndecoratedOptions identifier: ClassVar[str] = "acme.undecorated" + @property + def level(self) -> int: + return self.configuration.level + with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): CORE_AND_EXTENSIONS.extended_with(Undecorated) +@dataclass(frozen=True) +class ClosedOptions: + inner: CodecEntity + + def test_error_a_nested_field_admits_opaque() -> None: # What the field holds when the inner name is out of scope. @dataclass(frozen=True) class Closed(BytesBytesCodec): - inner: CodecEntity + configuration: ClosedOptions identifier: ClassVar[str] = "acme.closed" variable_size: ClassVar[bool] = False + @property + def inner(self) -> CodecEntity: + return self.configuration.inner + with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): CORE_AND_EXTENSIONS.extended_with(Closed) From c156049b57cf9cfdf9517f04b9b2c770e76df3a7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 21:27:14 +0200 Subject: [PATCH 091/107] refactor(zarr-metadata): the rules are the configuration record's own A configuration record subclasses `Configuration`, and the rules on its members are its `problems` method, yielding each as found: `BloscOptions(...).problems()` answers without an entity. The entity's constructor stops at the first; `coerce` asks the record before it builds anything and reports every one, so `create_unchecked` and the `object.__new__` behind it are gone. A family's rule about its name -- `r` a multiple of 8 -- is the entity's `name_problems(name)`, a classmethod, since a name is not configuration. The module-level rule functions, their quoted forward references and positional-only markers, and the `problems = fn` bindings go with them. No verdict or problem changes over the corpus. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_entity.py | 144 +++++++++--------- .../v3/chunk_grid/rectilinear.py | 34 ++--- .../zarr_metadata/v3/chunk_grid/regular.py | 20 +-- .../v3/chunk_key_encoding/default.py | 3 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 3 +- .../src/zarr_metadata/v3/codec/blosc.py | 51 +++---- .../src/zarr_metadata/v3/codec/bytes.py | 3 +- .../src/zarr_metadata/v3/codec/cast_value.py | 3 +- .../src/zarr_metadata/v3/codec/gzip.py | 16 +- .../zarr_metadata/v3/codec/scale_offset.py | 28 ++-- .../v3/codec/sharding_indexed.py | 20 +-- .../src/zarr_metadata/v3/codec/transpose.py | 28 ++-- .../src/zarr_metadata/v3/codec/zstd.py | 20 ++- .../zarr_metadata/v3/data_type/_families.py | 21 ++- .../src/zarr_metadata/v3/data_type/raw.py | 24 ++- .../src/zarr_metadata/v3/data_type/struct.py | 64 ++++---- .../src/zarr_metadata/v3/entity.py | 49 +++--- .../zarr-metadata/tests/test_public_api.py | 1 + .../tests/v3/test_acme_affine.py | 30 ++-- .../tests/v3/test_acme_decimal.py | 41 +++-- .../zarr-metadata/tests/v3/test_entities.py | 3 +- .../tests/v3/test_extension_api.py | 118 +++++++------- 22 files changed, 360 insertions(+), 364 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 967f9351a1..7ed7afde4d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -6,10 +6,10 @@ configuration against them, member by member, with `_typed_json`. A field typed `CodecEntity | Opaque` holds another entity, read through the scope the containing one is read in. Everything finer than a type -- -a bound, a rule about a member, members read together -- is a function -of the entity's instance that yields problems as it finds them, bound -on the class as `problems`; the constructor stops at the first, -`coerce` reports every one. +a bound, a rule about a member, members read together -- is the +record's own `problems`, which yields them as it finds them; the +constructor stops at the first, `coerce` reports every one, and a +reader may ask a record before building anything. What an entity writes follows from the same fields: `to_json` is written once here, the parser's inverse over each field's annotation. @@ -31,7 +31,16 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Mapping from dataclasses import dataclass, is_dataclass, replace -from typing import TYPE_CHECKING, ClassVar, Final, Literal, TypeAlias, TypeVar, cast, get_args +from typing import ( + TYPE_CHECKING, + ClassVar, + Final, + Literal, + TypeAlias, + TypeVar, + cast, + get_args, +) from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._typed_json import ( @@ -210,16 +219,16 @@ def canonical(self) -> Self: def unreadable(cls: type[MetadataEntity]) -> str | None: """Why `coerce` could not read an instance of `cls`; None if it can. - An entity's fields are `configuration`, a record dataclass of its - members, and at most one field the envelope's name fills. Things + An entity's fields are `configuration`, a `Configuration` record of + its members, and at most one field the envelope's name fills. Things that type-check cleanly and then go wrong somewhere that will not name the class: a field of any other name; a configuration that is - not a record, or a member of it whose annotation is not a shape JSON - takes; a `__post_init__` of the entity's own, whose rules `coerce` - would never ask; and a class variable a base annotates and nothing - sets -- `identifier` for every entity, `bounds` for an integer type - -- which the first lookup would fail. Registration asks, and refuses - the class with the answer. + not a `Configuration`, or a member of it whose annotation is not a + shape JSON takes; a `__post_init__` of the entity's own, whose rules + `coerce` would never ask; and a class variable a base annotates and + nothing sets -- `identifier` for every entity, `bounds` for an + integer type -- which the first lookup would fail. Registration asks, + and refuses the class with the answer. """ try: hints = field_hints(cls) @@ -235,13 +244,10 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: ) record = hints.get("configuration") if record is not None: - if ( - not (isinstance(record, type) and is_dataclass(record)) - or nested_kind(record) is not None - ): + if not (isinstance(record, type) and issubclass(record, Configuration)): return ( f"{cls.__name__}: configuration is annotated {record!r}; annotate it with a frozen " - "dataclass of the members, one field per configuration key" + "dataclass subclassing Configuration, one field per configuration member" ) try: members = field_hints(record) @@ -267,9 +273,9 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: ) if "__post_init__" in vars(cls): return ( - f"{cls.__name__} defines __post_init__; write its rules as a function of the " - "instance that yields problems and bind it as `problems = `: the " - "constructor stops at the first problem it yields, `coerce` reports every one" + f"{cls.__name__} defines __post_init__; write its rules as `problems` on its " + "Configuration record, yielding each: the constructor stops at the first, `coerce` " + "reports every one" ) annotated = declared_class_vars(cls) missing = sorted(name for name in annotated if not hasattr(cls, name)) @@ -407,6 +413,25 @@ def write(entity: MetadataEntity) -> dict[str, JSONValue]: return _Plan(from_name, parser(record, _nested_field), write, required) +@dataclass(frozen=True) +class Configuration: + """What an entity is configured with: a record of its members, and the rules on them. + + A frozen dataclass whose fields are the configuration's members, + each a shape JSON takes; the entity names it in its `configuration` + field. `problems` is where everything finer than a type goes -- a + bound, a rule about one member, members read together -- yielding + each problem as it is found, located relative to the configuration. + A reader stops at the first or collects them all, as it needs: the + entity's constructor stops at the first, `coerce` reports every one, + and `BloscOptions(...).problems()` answers without an entity at all. + """ + + def problems(self) -> Iterator[ValidationProblem]: + """Every reason these values are not allowed, yielded as found. Default: none.""" + yield from () + + @dataclass(frozen=True) class MetadataEntity(ABC): """One named entity, coerced from its metadata. @@ -426,13 +451,12 @@ class MetadataEntity(ABC): mapping instead: `MappingProxyType` is unhashable too, and anything else stops `json.dumps` from serializing what `to_json` returns. - A subclass writes its fields; where the spec has something to say - beyond their types, a function of the instance that yields problems, - bound as `problems` -- so `BloscCodec(clevel=99)` raises on the - first, and `coerce` reports every one instead; `to_json`, a literal - of its JSON type; and `canonical` where two spellings of its members - mean the same. `coerce` is written once here, against what the - fields say. + A subclass names its configuration record, a `Configuration` whose + `problems` holds what the spec says beyond the members' types -- so + `BloscCodec(BloscOptions(clevel=99))` raises on the first, and + `coerce` reports every one instead -- and writes `canonical` where + two spellings of its members mean the same. `coerce` and `to_json` + are written once here, against what the record says. """ identifier: ClassVar[str] @@ -443,40 +467,28 @@ class MetadataEntity(ABC): an invented identifier that no real name can collide with. """ - def problems(self, /) -> Iterator[ValidationProblem]: - """Every reason this entity's values are not allowed, yielded as found. + @classmethod + def name_problems(cls, name: str) -> Iterator[ValidationProblem]: + """Why `name`, which `accepts` claimed, is not a well-formed name of this family. - The entity's own rules -- a bound, a rule about one member, - members read together -- written as a function of the instance - and bound on the class: `problems = blosc_problems`. Locations - are relative to the configuration. A consumer stops at the first - or collects them all, as it needs: the constructor stops at the - first, `coerce` collects every one. Default: none. + For a family, whose names carry data -- `r` -- and which claims + a malformed member so that it is reported rather than waved + through as an unknown extension. Locations are relative to the + entity: `()`. Default: none, for an entity of one name. """ yield from () def __post_init__(self) -> None: - """Refuse the first problem `problems` finds, so `BloscCodec(clevel=99)` raises.""" - first = next(self.problems(), None) + """Refuse the first problem the rules find, so `BloscCodec(BloscOptions(clevel=99))` raises.""" + plan = _plan(type(self)) + name = self.identifier if plan.from_name is None else getattr(self, plan.from_name) + first = next(type(self).name_problems(name), None) + configuration = getattr(self, "configuration", None) + if first is None and isinstance(configuration, Configuration): + first = next(configuration.problems(), None) if first is not None: raise MetadataValidationError((first,)) - @classmethod - def create_unchecked(cls, **members: object) -> Self: - """The record `cls(**members)` would build, without asking `problems`. - - The constructor is the checked way to build an entity, and stops - at the first problem; this is for a reader that judges - afterwards and wants every one, as `coerce` does -- it asks - `problems` itself and reports what it yields. The members are - the caller's promise: nothing here checks their names or types, - which the constructor does. - """ - entity = object.__new__(cls) - for name, value in members.items(): - object.__setattr__(entity, name, value) - return entity - @classmethod def accepts(cls, name: str) -> bool: """Whether `name` denotes this entity. @@ -536,22 +548,17 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # An unknown key is survivable; a member that could not be # read is a hole, and judging around it would be guessing. return None, found - entity = cls.create_unchecked(**members) - # A problem about the member the envelope's name carries is - # about the entity, and lands on it rather than under a - # configuration the document does not have. - refused = within( - (), - tuple( - ValidationProblem((), entry.message, entry.kind) - if len(entry.loc) != 0 and entry.loc[0] == plan.from_name - else entry - for entry in entity.problems() - ), + # The rules, asked of the name and of the record before anything + # is built: a name problem lands on the entity, a configuration + # problem under the configuration. + record = members.get("configuration") + refused = ( + *cls.name_problems(name), + *(within((), tuple(record.problems())) if isinstance(record, Configuration) else ()), ) if len(refused) != 0: # Values the spec disallows: reported rather than raised, - # every one, located under the configuration. + # every one. return None, (*found, *refused) if any(entry.kind != "unknown_key" for entry in reading.nested): # A contained entity could not be read. This entity's own @@ -560,7 +567,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # entity that would be asked composition questions it cannot # answer. return None, found - return entity, found + return cls(**members), found def with_configuration(self, **changes: object) -> Self: """This entity with these configuration members changed. @@ -762,6 +769,7 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "ChunkKeyEncodingEntity", "CodecEntity", "Coerced", + "Configuration", "DataTypeEntity", "Loc", "MetadataEntity", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index f85645cd2e..b9f57a8c10 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -12,6 +12,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, + Configuration, Loc, is_integer, problem, @@ -167,28 +168,27 @@ def _axis_lengths(spec: RectilinearDimSpec) -> frozenset[int] | None: @dataclass(frozen=True) -class RectilinearChunkGridOptions: +class RectilinearChunkGridOptions(Configuration): """What a `rectilinear` grid is configured with.""" kind: Literal["inline"] chunk_shapes: tuple[RectilinearDimSpec, ...] - -def rectilinear_problems(grid: "RectilinearChunkGrid", /) -> "Iterator[ValidationProblem]": - """Every extent, and every run's length and count, is at least 1.""" - for axis, spec in enumerate(grid.chunk_shapes): - if isinstance(spec, int): - if spec < 1: - yield _not_positive(("chunk_shapes", axis), spec) - continue - for index, entry in enumerate(spec): - if isinstance(entry, int): - if entry < 1: - yield _not_positive(("chunk_shapes", axis, index), entry) + def problems(self) -> "Iterator[ValidationProblem]": + """Every extent, and every run's length and count, is at least 1.""" + for axis, spec in enumerate(self.chunk_shapes): + if isinstance(spec, int): + if spec < 1: + yield _not_positive(("chunk_shapes", axis), spec) continue - for position, value in enumerate(entry): - if value < 1: - yield _not_positive(("chunk_shapes", axis, index, position), value) + for index, entry in enumerate(spec): + if isinstance(entry, int): + if entry < 1: + yield _not_positive(("chunk_shapes", axis, index), entry) + continue + for position, value in enumerate(entry): + if value < 1: + yield _not_positive(("chunk_shapes", axis, index, position), value) @dataclass(frozen=True) @@ -199,8 +199,6 @@ class RectilinearChunkGrid(ChunkGridEntity): identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME - problems = rectilinear_problems - @property def kind(self) -> Literal["inline"]: return self.configuration.kind diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index e725b5606b..b4b6950757 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -12,6 +12,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ChunkGridEntity, + Configuration, problem, ) from zarr_metadata.v3._parts import ChunkGrid @@ -61,18 +62,19 @@ class RegularChunkGridObject(TypedDict, closed=True): @dataclass(frozen=True) -class RegularChunkGridOptions: +class RegularChunkGridOptions(Configuration): """What a `regular` grid is configured with.""" chunk_shape: tuple[int, ...] - -def regular_problems(grid: "RegularChunkGrid", /) -> "Iterator[ValidationProblem]": - for index, extent in enumerate(grid.chunk_shape): - if extent < 1: - yield ValidationProblem( - ("chunk_shape", index), f"expected an integer >= 1, got {extent}", "invalid_value" - ) + def problems(self) -> "Iterator[ValidationProblem]": + for index, extent in enumerate(self.chunk_shape): + if extent < 1: + yield ValidationProblem( + ("chunk_shape", index), + f"expected an integer >= 1, got {extent}", + "invalid_value", + ) @dataclass(frozen=True) @@ -83,8 +85,6 @@ class RegularChunkGrid(ChunkGridEntity): identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME - problems = regular_problems - @property def chunk_shape(self) -> tuple[int, ...]: return self.configuration.chunk_shape diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index e62f084ef7..463bf8bb81 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -15,6 +15,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( ChunkKeyEncodingEntity, + Configuration, ) DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" @@ -71,7 +72,7 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class DefaultChunkKeyEncodingOptions: +class DefaultChunkKeyEncodingOptions(Configuration): """What the `default` encoding is configured with.""" separator: DefaultChunkKeyEncodingSeparator | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 3f7d7698d1..d3aedd1961 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -21,6 +21,7 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.v3._entity import ( ChunkKeyEncodingEntity, + Configuration, ) V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" @@ -77,7 +78,7 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): @dataclass(frozen=True) -class V2ChunkKeyEncodingOptions: +class V2ChunkKeyEncodingOptions(Configuration): """What the `v2` encoding is configured with.""" separator: V2ChunkKeyEncodingSeparator | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index ca6e57c65d..3c19c10beb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -13,6 +13,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( BytesBytesCodec, + Configuration, ) if TYPE_CHECKING: @@ -90,7 +91,7 @@ class BloscCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BloscOptions: +class BloscOptions(Configuration): """What `blosc` is configured with.""" cname: BloscCName @@ -99,35 +100,34 @@ class BloscOptions: blocksize: int typesize: int | UNSET = UNSET + def problems(self) -> "Iterator[ValidationProblem]": + """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. -def blosc_problems(codec: "BloscCodec", /) -> "Iterator[ValidationProblem]": - """Bounds on `clevel` and `blocksize`; `typesize` against `shuffle`. - - Under `noshuffle` the spec says of `typesize` that "the value is - ignored", and `canonical` drops it; under either shuffle it is - required, and positive. - """ - if not 0 <= codec.clevel <= 9: - yield ValidationProblem( - ("clevel",), f"expected an integer in [0, 9], got {codec.clevel}", "invalid_value" - ) - if codec.blocksize < 0: - yield ValidationProblem( - ("blocksize",), f"expected an integer >= 0, got {codec.blocksize}", "invalid_value" - ) - if codec.shuffle != BLOSC_NO_SHUFFLE: - if codec.typesize is UNSET: + Under `noshuffle` the spec says of `typesize` that "the value is + ignored", and `canonical` drops it; under either shuffle it is + required, and positive. + """ + if not 0 <= self.clevel <= 9: yield ValidationProblem( - ("typesize",), - f"typesize is required when shuffle is {codec.shuffle!r}", - "missing_key", + ("clevel",), f"expected an integer in [0, 9], got {self.clevel}", "invalid_value" ) - elif codec.typesize < 1: + if self.blocksize < 0: yield ValidationProblem( - ("typesize",), - f"expected a positive integer, got {codec.typesize}", - "invalid_value", + ("blocksize",), f"expected an integer >= 0, got {self.blocksize}", "invalid_value" ) + if self.shuffle != BLOSC_NO_SHUFFLE: + if self.typesize is UNSET: + yield ValidationProblem( + ("typesize",), + f"typesize is required when shuffle is {self.shuffle!r}", + "missing_key", + ) + elif self.typesize < 1: + yield ValidationProblem( + ("typesize",), + f"expected a positive integer, got {self.typesize}", + "invalid_value", + ) @dataclass(frozen=True) @@ -146,7 +146,6 @@ class BloscCodec(BytesBytesCodec): # Every member is required but `typesize`, which only means something # when shuffling; `blosc_problems` is where that conditional lives. - problems = blosc_problems @property def cname(self) -> BloscCName: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index ace8a64a17..e09c08899e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -13,6 +13,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ArrayBytesCodec, + Configuration, DataTypeEntity, problem, ) @@ -79,7 +80,7 @@ class BytesCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class BytesOptions: +class BytesOptions(Configuration): """What `bytes` is configured with.""" endian: Endianness | UNSET = UNSET diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 313fbfecc5..4285ef6a27 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -14,6 +14,7 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( ArrayArrayCodec, + Configuration, DataTypeEntity, Opaque, ) @@ -125,7 +126,7 @@ class CastValueCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class CastValueOptions: +class CastValueOptions(Configuration): """What `cast_value` is configured with.""" data_type: DataTypeEntity | Opaque diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 3cc333c00b..671442d017 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -12,6 +12,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( BytesBytesCodec, + Configuration, ) if TYPE_CHECKING: @@ -68,17 +69,16 @@ class GzipCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class GzipOptions: +class GzipOptions(Configuration): """What `gzip` is configured with.""" level: int - -def gzip_problems(codec: "GzipCodec", /) -> "Iterator[ValidationProblem]": - if not 0 <= codec.level <= 9: - yield ValidationProblem( - ("level",), f"expected an integer in [0, 9], got {codec.level}", "invalid_value" - ) + def problems(self) -> "Iterator[ValidationProblem]": + if not 0 <= self.level <= 9: + yield ValidationProblem( + ("level",), f"expected an integer in [0, 9], got {self.level}", "invalid_value" + ) @dataclass(frozen=True) @@ -90,8 +90,6 @@ class GzipCodec(BytesBytesCodec): identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True - problems = gzip_problems - @property def level(self) -> int: return self.configuration.level diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index a1fb70f459..622967253c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -14,6 +14,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ArrayArrayCodec, + Configuration, ) from zarr_metadata.v3._parts import ArrayParts @@ -75,25 +76,24 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ScaleOffsetOptions: +class ScaleOffsetOptions(Configuration): """What `scale_offset` is configured with.""" offset: JSONValue | UNSET = UNSET scale: JSONValue | UNSET = UNSET + def problems(self) -> "Iterator[ValidationProblem]": + """Each value is a scalar of the array's type, so neither is null. -def scale_offset_problems(codec: "ScaleOffsetCodec", /) -> "Iterator[ValidationProblem]": - """Each value is a scalar of the array's type, so neither is null. - - The registry says each is "JSON-encoded per the input array's - fill-value rules", and no data type admits `null` as a fill value. - Which scalar it should be needs the data type, so that part is the - document's question, not this codec's. - """ - if codec.offset is None: - yield ValidationProblem(("offset",), "expected a scalar, got null", "invalid_value") - if codec.scale is None: - yield ValidationProblem(("scale",), "expected a scalar, got null", "invalid_value") + The registry says each is "JSON-encoded per the input array's + fill-value rules", and no data type admits `null` as a fill value. + Which scalar it should be needs the data type, so that part is the + document's question, not this codec's. + """ + if self.offset is None: + yield ValidationProblem(("offset",), "expected a scalar, got null", "invalid_value") + if self.scale is None: + yield ValidationProblem(("scale",), "expected a scalar, got null", "invalid_value") @dataclass(frozen=True) @@ -110,8 +110,6 @@ class ScaleOffsetCodec(ArrayArrayCodec): identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME variable_size: ClassVar[bool] = False - problems = scale_offset_problems - @property def offset(self) -> JSONValue | UNSET: return self.configuration.offset diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index f5aa1b701e..1f3a486337 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -16,6 +16,7 @@ from zarr_metadata.v3._entity import ( ArrayBytesCodec, CodecEntity, + Configuration, Opaque, problem, ) @@ -98,7 +99,7 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ShardingIndexedOptions: +class ShardingIndexedOptions(Configuration): """What `sharding_indexed` is configured with.""" chunk_shape: tuple[int, ...] @@ -106,13 +107,14 @@ class ShardingIndexedOptions: index_codecs: tuple[CodecEntity | Opaque, ...] index_location: ShardingIndexLocation | UNSET = UNSET - -def sharding_problems(codec: "ShardingIndexedCodec", /) -> "Iterator[ValidationProblem]": - for index, extent in enumerate(codec.chunk_shape): - if extent < 1: - yield ValidationProblem( - ("chunk_shape", index), f"expected an integer >= 1, got {extent}", "invalid_value" - ) + def problems(self) -> "Iterator[ValidationProblem]": + for index, extent in enumerate(self.chunk_shape): + if extent < 1: + yield ValidationProblem( + ("chunk_shape", index), + f"expected an integer >= 1, got {extent}", + "invalid_value", + ) @dataclass(frozen=True) @@ -129,8 +131,6 @@ class ShardingIndexedCodec(ArrayBytesCodec): identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME variable_size: ClassVar[bool] = True - problems = sharding_problems - @property def chunk_shape(self) -> tuple[int, ...]: return self.configuration.chunk_shape diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index a1f8160f3d..9a3ec65677 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -12,6 +12,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( ArrayArrayCodec, + Configuration, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -65,24 +66,23 @@ class TransposeCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class TransposeOptions: +class TransposeOptions(Configuration): """What `transpose` is configured with.""" order: tuple[int, ...] + def problems(self) -> "Iterator[ValidationProblem]": + """`order` must permute its own axes. -def transpose_problems(codec: "TransposeCodec", /) -> "Iterator[ValidationProblem]": - """`order` must permute its own axes. - - Whether it permutes the *array's* axes is a different question -- it - needs the array's rank -- and the rules layer asks that one. - """ - if sorted(codec.order) != list(range(len(codec.order))): - yield ValidationProblem( - ("order",), - f"expected a permutation of 0..{len(codec.order) - 1}, got {codec.order!r}", - "invalid_value", - ) + Whether it permutes the *array's* axes is a different question -- it + needs the array's rank -- and the rules layer asks that one. + """ + if sorted(self.order) != list(range(len(self.order))): + yield ValidationProblem( + ("order",), + f"expected a permutation of 0..{len(self.order) - 1}, got {self.order!r}", + "invalid_value", + ) @dataclass(frozen=True) @@ -94,8 +94,6 @@ class TransposeCodec(ArrayArrayCodec): identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME variable_size: ClassVar[bool] = False - problems = transpose_problems - @property def order(self) -> tuple[int, ...]: return self.configuration.order diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 4b802df965..c877e976ea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -15,6 +15,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( BytesBytesCodec, + Configuration, ) if TYPE_CHECKING: @@ -76,20 +77,19 @@ class ZstdCodecObject(TypedDict, closed=True): @dataclass(frozen=True) -class ZstdOptions: +class ZstdOptions(Configuration): """What `zstd` is configured with.""" level: int checksum: bool | UNSET = UNSET - -def zstd_problems(codec: "ZstdCodec", /) -> "Iterator[ValidationProblem]": - if not ZSTD_MIN_LEVEL <= codec.level <= ZSTD_MAX_LEVEL: - yield ValidationProblem( - ("level",), - f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {codec.level}", - "invalid_value", - ) + def problems(self) -> "Iterator[ValidationProblem]": + if not ZSTD_MIN_LEVEL <= self.level <= ZSTD_MAX_LEVEL: + yield ValidationProblem( + ("level",), + f"expected an integer in [{ZSTD_MIN_LEVEL}, {ZSTD_MAX_LEVEL}], got {self.level}", + "invalid_value", + ) @dataclass(frozen=True) @@ -101,8 +101,6 @@ class ZstdCodec(BytesBytesCodec): identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True - problems = zstd_problems - @property def level(self) -> int: return self.configuration.level diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index e9766aec66..762e498cd0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -19,6 +19,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + Configuration, DataTypeEntity, StorageClass, is_integer, @@ -168,21 +169,20 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP @dataclass(frozen=True) -class NumpyTimeOptions: +class NumpyTimeOptions(Configuration): """What a numpy time type is configured with: a unit, and how many of it one tick is.""" unit: NumpyTimeUnit scale_factor: int - -def numpy_time_problems(data_type: NumpyTimeDataType, /) -> Iterator[ValidationProblem]: - if not 1 <= data_type.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: - yield ValidationProblem( - ("scale_factor",), - f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " - f"got {data_type.scale_factor}", - "invalid_value", - ) + def problems(self) -> Iterator[ValidationProblem]: + if not 1 <= self.scale_factor <= NUMPY_TIME_MAX_SCALE_FACTOR: + yield ValidationProblem( + ("scale_factor",), + f"expected an integer in [1, {NUMPY_TIME_MAX_SCALE_FACTOR}], " + f"got {self.scale_factor}", + "invalid_value", + ) @dataclass(frozen=True) @@ -197,7 +197,6 @@ class NumpyTimeDataType(DataTypeEntity): configuration: NumpyTimeOptions scalar_storage: ClassVar[StorageClass] = "multi_byte" - problems = numpy_time_problems @property def unit(self) -> NumpyTimeUnit: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 13db2e357e..328c4eee84 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -84,18 +84,6 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: ] -def raw_bytes_problems(data_type: "RawBytesDataType", /) -> "Iterator[ValidationProblem]": - """This family's validity is in its name, not in a configuration. - - "raw bits, variable size given by *, limited to be a multiple of 8" - -- and zero bits is not a data type. - """ - try: - raw_bytes_dtype_name(data_type.data_type_name) - except ValueError as error: - yield ValidationProblem((), str(error), "invalid_value") - - @dataclass(frozen=True) class RawBytesDataType(DataTypeEntity): """An `r` raw-bytes data type, coerced from its metadata. @@ -126,7 +114,17 @@ def accepts(cls, name: str) -> bool: """ return RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None - problems = raw_bytes_problems + @classmethod + def name_problems(cls, name: str) -> "Iterator[ValidationProblem]": + """This family's validity is in its name, not in a configuration. + + "raw bits, variable size given by *, limited to be a multiple of 8" + -- and zero bits is not a data type. + """ + try: + raw_bytes_dtype_name(name) + except ValueError as error: + yield ValidationProblem((), str(error), "invalid_value") def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """One byte value per byte of the scalar. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 61502d61b0..b6b1ddcb6c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -14,6 +14,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( + Configuration, DataTypeEntity, Loc, Opaque, @@ -99,43 +100,44 @@ class StructFieldComponent: @dataclass(frozen=True) -class StructOptions: +class StructOptions(Configuration): """What `struct` is configured with.""" fields: tuple[StructFieldComponent, ...] + def problems(self) -> "Iterator[ValidationProblem]": + """Names exist, are non-empty and distinct; types are fixed-size. -def struct_problems(data_type: "StructDataType", /) -> "Iterator[ValidationProblem]": - """Names exist, are non-empty and distinct; types are fixed-size. - - A fill value addresses fields by name, and a record's layout is not - determined by a variable-length field. Nothing about a field type's - own values: it is an entity, so it exists only if those are allowed. - """ - if len(data_type.fields) == 0: - yield ValidationProblem(("fields",), "expected at least one struct field", "invalid_value") - seen: dict[str, int] = {} - for index, field in enumerate(data_type.fields): - if field.name == "": - yield ValidationProblem( - ("fields", index, "name"), "expected a non-empty field name", "invalid_value" - ) - first = seen.setdefault(field.name, index) - if first != index: - yield ValidationProblem( - ("fields", index, "name"), - f"duplicate field name {field.name!r}, already used by field {first}", - "invalid_value", - ) - if ( - isinstance(field.data_type, DataTypeEntity) - and field.data_type.storage_class() == "variable_length" - ): + A fill value addresses fields by name, and a record's layout is not + determined by a variable-length field. Nothing about a field type's + own values: it is an entity, so it exists only if those are allowed. + """ + if len(self.fields) == 0: yield ValidationProblem( - ("fields", index, "data_type"), - "struct fields must use fixed-size data types", - "invalid_value", + ("fields",), "expected at least one struct field", "invalid_value" ) + seen: dict[str, int] = {} + for index, field in enumerate(self.fields): + if field.name == "": + yield ValidationProblem( + ("fields", index, "name"), "expected a non-empty field name", "invalid_value" + ) + first = seen.setdefault(field.name, index) + if first != index: + yield ValidationProblem( + ("fields", index, "name"), + f"duplicate field name {field.name!r}, already used by field {first}", + "invalid_value", + ) + if ( + isinstance(field.data_type, DataTypeEntity) + and field.data_type.storage_class() == "variable_length" + ): + yield ValidationProblem( + ("fields", index, "data_type"), + "struct fields must use fixed-size data types", + "invalid_value", + ) @dataclass(frozen=True) @@ -152,8 +154,6 @@ class StructDataType(DataTypeEntity): identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" - problems = struct_problems - @property def fields(self) -> tuple[StructFieldComponent, ...]: return self.configuration.fields diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index e3e4e0a464..7dbeb8777b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -40,10 +40,9 @@ kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity` or `StorageTransformerEntity`; declare the configuration as a frozen -dataclass of its members and name it in the entity's one field, -`configuration`; write every rule finer than a type as a function of the -instance that yields problems, and bind it as `problems`; add the class -to a scope. Complete; runnable given a `document`: +`Configuration` of its members, with every rule finer than a type in +its `problems`, and name it in the entity's one field, `configuration`; +add the class to a scope. Complete; runnable given a `document`: from collections.abc import Iterator from dataclasses import dataclass @@ -54,21 +53,21 @@ CORE_AND_EXTENSIONS, UNSET, BytesBytesCodec, + Configuration, ValidationProblem, ) @dataclass(frozen=True) # the fields are the schema; frozen, so a configuration is a value - class AcmeLz4Options: + class AcmeLz4Options(Configuration): acceleration: int | UNSET = UNSET # optional: absent reads as UNSET - def acme_lz4_problems(codec: "AcmeLz4Codec", /) -> Iterator[ValidationProblem]: - acceleration = codec.configuration.acceleration - if acceleration is not UNSET and not 1 <= acceleration <= 65537: - yield ValidationProblem( - ("acceleration",), - f"expected an integer in [1, 65537], got {acceleration}", - "invalid_value", - ) + def problems(self) -> Iterator[ValidationProblem]: + if self.acceleration is not UNSET and not 1 <= self.acceleration <= 65537: + yield ValidationProblem( + ("acceleration",), + f"expected an integer in [1, 65537], got {self.acceleration}", + "invalid_value", + ) @dataclass(frozen=True) class AcmeLz4Codec(BytesBytesCodec): @@ -76,7 +75,6 @@ class AcmeLz4Codec(BytesBytesCodec): identifier: ClassVar[str] = "acme.lz4" variable_size: ClassVar[bool] = True # a compressor: its output length is not fixed - problems = acme_lz4_problems SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec) validate_array_metadata_v3(document, context=SCOPE) @@ -104,17 +102,18 @@ class AcmeLz4Codec(BytesBytesCodec): any construction is. Everything finer than a type -- a bound, a rule about one member, members -read together -- is a function of the instance that yields +read together -- is the record's `problems`, which yields `ValidationProblem(loc, message, kind)` as it finds each, in plain -code, bound on the class as `problems`. Locations are relative to the -configuration, and `kind` is `"invalid_value"` for a value rule. The -constructor stops at the first problem it yields, so -`AcmeLz4Codec(acceleration=0)` raises `MetadataValidationError`; -`coerce` runs it to the end and reports every problem in the document. -`create_unchecked(**members)` builds the record without the check, for -a reader that judges afterwards with `problems` and wants every one. -It runs only on an entity whose members all read: a member of the wrong -type is reported and the entity is not built. +code. Locations are relative to the configuration, and `kind` is +`"invalid_value"` for a value rule. The entity's constructor stops at +the first problem it yields, so `AcmeLz4Codec(AcmeLz4Options(acceleration=0))` +raises `MetadataValidationError`; `coerce` runs it to the end and +reports every problem in the document; a reader with a record asks +`options.problems()` directly and stops or collects. It runs only on a +configuration whose members all read: a member of the wrong type is +reported and the entity is not built. A family's rule about its name -- +`r` a multiple of 8 -- is the entity's `name_problems(name)`, a +classmethod, located on the entity. **What an entity answers for itself**, beyond its configuration. `to_json` is written once in the base, from the record: the bare name when every @@ -198,6 +197,7 @@ class AcmeLz4Codec(BytesBytesCodec): ChunkKeyEncodingEntity, CodecEntity, Coerced, + Configuration, DataTypeEntity, Loc, MetadataEntity, @@ -234,6 +234,7 @@ class AcmeLz4Codec(BytesBytesCodec): "CodecEntity", "Coerced", "ComplexDataType", + "Configuration", "Context", "DataTypeEntity", "Extents", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index c78410085c..542b697960 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -295,6 +295,7 @@ def test_all_is_grouped_and_unique() -> None: "Extents", "Context", "Coerced", + "Configuration", "ChunkGrid", "ArrayParts", "ArrayDocumentV3", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 99e1eb4e21..76be17d5c4 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -26,6 +26,7 @@ ArrayArrayCodec, ArrayDocumentV3, ArrayParts, + Configuration, DataTypeEntity, MetadataValidationError, Opaque, @@ -50,23 +51,27 @@ class AcmeAffineObject(TypedDict, closed=True): must_understand: NotRequired[bool] -def acme_affine_problems(codec: AcmeAffineCodec, /) -> Iterator[ValidationProblem]: - if codec.scale == 0: - yield ValidationProblem(("scale",), "expected a non-zero number, got 0", "invalid_value") - if isinstance(codec.dtype, DataTypeEntity) and codec.dtype.storage_class() == "variable_length": - yield ValidationProblem( - ("dtype",), - f"expected a fixed-size data type, got {type(codec.dtype).identifier!r}", - "invalid_value", - ) - - @dataclass(frozen=True) -class AcmeAffineOptions: +class AcmeAffineOptions(Configuration): scale: float offset: float | UNSET = UNSET dtype: DataTypeEntity | Opaque | UNSET = UNSET + def problems(self) -> Iterator[ValidationProblem]: + if self.scale == 0: + yield ValidationProblem( + ("scale",), "expected a non-zero number, got 0", "invalid_value" + ) + if ( + isinstance(self.dtype, DataTypeEntity) + and self.dtype.storage_class() == "variable_length" + ): + yield ValidationProblem( + ("dtype",), + f"expected a fixed-size data type, got {type(self.dtype).identifier!r}", + "invalid_value", + ) + @dataclass(frozen=True) class AcmeAffineCodec(ArrayArrayCodec): @@ -76,7 +81,6 @@ class AcmeAffineCodec(ArrayArrayCodec): identifier: ClassVar[str] = "acme.affine" variable_size: ClassVar[bool] = False - problems = acme_affine_problems @property def scale(self) -> float: diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 8a658448e3..16c781fe1a 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -19,6 +19,7 @@ from zarr_metadata.v3.entity import ( + Configuration, DataTypeEntity, Loc, MetadataValidationError, @@ -70,30 +71,29 @@ class AcmeDecimal(TypedDict, closed=True): ] -def acme_decimal_problems(data_type: AcmeDecimalDataType, /) -> Iterator[ValidationProblem]: - if not 1 <= data_type.precision <= ACME_DECIMAL_MAX_PRECISION: - yield ValidationProblem( - ("precision",), - f"expected an integer in [1, {ACME_DECIMAL_MAX_PRECISION}], got {data_type.precision}", - "invalid_value", - ) - if data_type.scale < 0: - yield ValidationProblem( - ("scale",), f"expected an integer >= 0, got {data_type.scale}", "invalid_value" - ) - elif data_type.scale > data_type.precision: - yield ValidationProblem( - ("scale",), - f"expected an integer <= precision ({data_type.precision}), got {data_type.scale}", - "invalid_value", - ) - - @dataclass(frozen=True) -class AcmeDecimalOptions: +class AcmeDecimalOptions(Configuration): precision: int scale: int + def problems(self) -> Iterator[ValidationProblem]: + if not 1 <= self.precision <= ACME_DECIMAL_MAX_PRECISION: + yield ValidationProblem( + ("precision",), + f"expected an integer in [1, {ACME_DECIMAL_MAX_PRECISION}], got {self.precision}", + "invalid_value", + ) + if self.scale < 0: + yield ValidationProblem( + ("scale",), f"expected an integer >= 0, got {self.scale}", "invalid_value" + ) + elif self.scale > self.precision: + yield ValidationProblem( + ("scale",), + f"expected an integer <= precision ({self.precision}), got {self.scale}", + "invalid_value", + ) + @dataclass(frozen=True) class AcmeDecimalDataType(DataTypeEntity): @@ -103,7 +103,6 @@ class AcmeDecimalDataType(DataTypeEntity): identifier: ClassVar[str] = ACME_DECIMAL_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "multi_byte" - problems = acme_decimal_problems @property def precision(self) -> int: diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 95a3ab54b7..94c86ccfe0 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -82,6 +82,7 @@ ChunkGridEntity, ChunkKeyEncodingEntity, CodecEntity, + Configuration, DataTypeEntity, MetadataEntity, StorageTransformerEntity, @@ -737,7 +738,7 @@ def test_the_fail_fast_reader_refuses_a_member_it_would_drop() -> None: # A storage transformer: the one extension point nothing in the package # models, so the only way to reach it is to register one. @dataclasses.dataclass(frozen=True) -class AcmeShardCacheOptions: +class AcmeShardCacheOptions(Configuration): verbose: bool | UNSET = UNSET diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 3bd3e23d84..0f9fa8716b 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -30,6 +30,7 @@ ChunkGridEntity, ChunkKeyEncodingEntity, CodecEntity, + Configuration, Context, DataTypeEntity, IntegerDataType, @@ -47,19 +48,18 @@ ACME_MAX_ACCELERATION = 65537 -def acme_lz4_problems(codec: AcmeLz4Codec, /) -> Iterator[ValidationProblem]: - if codec.acceleration is not UNSET and not 1 <= codec.acceleration <= ACME_MAX_ACCELERATION: - yield ValidationProblem( - ("acceleration",), - f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {codec.acceleration}", - "invalid_value", - ) - - @dataclass(frozen=True) -class AcmeLz4Options: +class AcmeLz4Options(Configuration): acceleration: int | UNSET = UNSET + def problems(self) -> Iterator[ValidationProblem]: + if self.acceleration is not UNSET and not 1 <= self.acceleration <= ACME_MAX_ACCELERATION: + yield ValidationProblem( + ("acceleration",), + f"expected an integer in [1, {ACME_MAX_ACCELERATION}], got {self.acceleration}", + "invalid_value", + ) + @dataclass(frozen=True) class AcmeLz4Codec(BytesBytesCodec): @@ -69,7 +69,6 @@ class AcmeLz4Codec(BytesBytesCodec): identifier: ClassVar[str] = "acme.lz4" variable_size: ClassVar[bool] = True - problems = acme_lz4_problems @property def acceleration(self) -> int | UNSET: @@ -187,7 +186,7 @@ def test_the_entity_layer_answers_what_a_reader_needs() -> None: @dataclass(frozen=True) -class DefaultedOptions: +class DefaultedOptions(Configuration): level: int | UNSET = 3 @@ -294,14 +293,6 @@ class Int24DataType(IntegerDataType): ACME_FIXED_PATTERN = re.compile(r"acme\.fixed(\d+)") -def acme_fixed_problems(data_type: AcmeFixedDataType, /) -> Iterator[ValidationProblem]: - match = ACME_FIXED_PATTERN.fullmatch(data_type.data_type_name) - if match is not None and int(match.group(1)) % 8 != 0: - yield ValidationProblem( - ("data_type_name",), "expected a width that is a multiple of 8", "invalid_value" - ) - - @dataclass(frozen=True) class AcmeFixedDataType(DataTypeEntity): """`acme.fixedN`, a fixed-width type for every N.""" @@ -310,7 +301,12 @@ class AcmeFixedDataType(DataTypeEntity): identifier: ClassVar[str] = "acme.fixed" scalar_storage: ClassVar[StorageClass] = "multi_byte" - problems = acme_fixed_problems + + @classmethod + def name_problems(cls, name: str) -> Iterator[ValidationProblem]: + match = ACME_FIXED_PATTERN.fullmatch(name) + if match is not None and int(match.group(1)) % 8 != 0: + yield ValidationProblem((), "expected a width that is a multiple of 8", "invalid_value") @classmethod def accepts(cls, name: str) -> bool: @@ -345,7 +341,7 @@ def test_a_third_party_can_register_a_family() -> None: @dataclass(frozen=True) -class StructuredOptions: +class StructuredOptions(Configuration): inner: object @@ -370,7 +366,7 @@ def inner(self) -> object: # A third-party codec that contains another codec. @dataclass(frozen=True) -class AcmeWrapperOptions: +class AcmeWrapperOptions(Configuration): inner: CodecEntity | Opaque @@ -451,7 +447,7 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: @dataclass(frozen=True) -class AcmeFramedOptions: +class AcmeFramedOptions(Configuration): inner: CodecEntity | Opaque frame: int | UNSET = UNSET @@ -494,7 +490,7 @@ def canonical(self) -> Self: @dataclass(frozen=True) -class VagueOptions: +class VagueOptions(Configuration): inner: MetadataEntity | Opaque @@ -538,18 +534,19 @@ class AcmeBlockObject(TypedDict, closed=True): must_understand: NotRequired[bool] -# A third-party rule about a member: a function of the instance. -def acme_block_problems(codec: AcmeBlockCodec, /) -> Iterator[ValidationProblem]: - if codec.block < 1 or codec.block & (codec.block - 1) != 0: - yield ValidationProblem( - ("block",), f"expected a power of two, got {codec.block}", "invalid_value" - ) +# A third-party rule about a member: the record's own. @dataclass(frozen=True) -class AcmeBlockOptions: +class AcmeBlockOptions(Configuration): block: int + def problems(self) -> Iterator[ValidationProblem]: + if self.block < 1 or self.block & (self.block - 1) != 0: + yield ValidationProblem( + ("block",), f"expected a power of two, got {self.block}", "invalid_value" + ) + @dataclass(frozen=True) class AcmeBlockCodec(BytesBytesCodec): @@ -560,15 +557,14 @@ class AcmeBlockCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.block" variable_size: ClassVar[bool] = False - problems = acme_block_problems @property def block(self) -> int: return self.configuration.block -def test_a_rule_about_a_member_is_a_function_of_the_instance() -> None: - # The rule runs on the typed members and reports relative to the +def test_a_rule_about_a_member_is_the_record_s_own() -> None: + # The rule runs on the typed record and reports relative to the # configuration; `coerce` runs it to the end and locates what it # yields in the document; the constructor stops at the first. scope = CORE_AND_EXTENSIONS.extended_with(AcmeBlockCodec) @@ -590,22 +586,21 @@ def test_a_rule_about_a_member_is_a_function_of_the_instance() -> None: assert [p.kind for p in problems] == ["invalid_type"] -def acme_range_problems(codec: AcmeRangeCodec, /) -> Iterator[ValidationProblem]: - if codec.low < 0: - yield ValidationProblem( - ("low",), f"expected an integer >= 0, got {codec.low}", "invalid_value" - ) - if codec.high < codec.low: - yield ValidationProblem( - ("high",), f"expected an integer >= low, got {codec.high}", "invalid_value" - ) - - @dataclass(frozen=True) -class AcmeRangeOptions: +class AcmeRangeOptions(Configuration): low: int high: int + def problems(self) -> Iterator[ValidationProblem]: + if self.low < 0: + yield ValidationProblem( + ("low",), f"expected an integer >= 0, got {self.low}", "invalid_value" + ) + if self.high < self.low: + yield ValidationProblem( + ("high",), f"expected an integer >= low, got {self.high}", "invalid_value" + ) + @dataclass(frozen=True) class AcmeRangeCodec(BytesBytesCodec): @@ -616,7 +611,6 @@ class AcmeRangeCodec(BytesBytesCodec): identifier: ClassVar[str] = "acme.range" variable_size: ClassVar[bool] = False - problems = acme_range_problems @property def low(self) -> int: @@ -628,9 +622,9 @@ def high(self) -> int: def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: - # One function, two consumers: the constructor takes the first - # problem it yields, `coerce` runs it to the end. A consumer holding - # an entity may run it too, and stop or collect as it likes. + # One method, three consumers: the entity's constructor takes the + # first problem it yields, `coerce` runs it to the end, and a reader + # with a record asks it directly, and stops or collects. with pytest.raises(MetadataValidationError) as caught: AcmeRangeCodec(AcmeRangeOptions(low=-1, high=-2)) assert [p.loc for p in caught.value.problems] == [("low",)] @@ -639,16 +633,14 @@ def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one CodecEntity, {"name": "acme.range", "configuration": {"low": -1, "high": -2}} ) assert [p.loc for p in problems] == [("configuration", "low"), ("configuration", "high")] - assert list(acme_range_problems(AcmeRangeCodec(AcmeRangeOptions(low=0, high=1)))) == [] - # A reader that wants every problem of a hand-built one builds the - # record without the check and asks. - record = AcmeRangeCodec.create_unchecked(configuration=AcmeRangeOptions(low=-1, high=-2)) - assert [p.loc for p in record.problems()] == [("low",), ("high",)] + assert list(AcmeRangeOptions(low=0, high=1).problems()) == [] + # A reader that wants every problem of a record asks the record. + assert [p.loc for p in AcmeRangeOptions(low=-1, high=-2).problems()] == [("low",), ("high",)] def test_error_an_entity_may_not_define_post_init() -> None: # `coerce` never runs it, so a rule written there would judge a - # hand-built entity and no document. + # hand-built entity and no document; the rules go on the record. @dataclass(frozen=True) class Checked(BytesBytesCodec): identifier: ClassVar[str] = "acme.checked" @@ -657,12 +649,12 @@ class Checked(BytesBytesCodec): def __post_init__(self) -> None: return None - with pytest.raises(TypeError, match="defines __post_init__; write its rules as a function"): + with pytest.raises(TypeError, match="defines __post_init__; write its rules as `problems`"): CORE_AND_EXTENSIONS.extended_with(Checked) @dataclass(frozen=True) -class LocalizedOptions: +class LocalizedOptions(Configuration): # `Local` is defined inside the test, so it is not here, where this # class's annotations resolve: the case the message is for. inner: Local # noqa: F821 # pyright: ignore[reportUndefinedVariable] @@ -718,7 +710,7 @@ def test_a_malformed_envelope_is_one_problem() -> None: @dataclass(frozen=True) -class AcmeSlottedOptions: +class AcmeSlottedOptions(Configuration): level: int @@ -754,7 +746,7 @@ class AcmeNoted(BytesBytesCodec): @dataclass(frozen=True) -class AcmeScaledOptions: +class AcmeScaledOptions(Configuration): scale: float @@ -793,7 +785,7 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: @dataclass(frozen=True) -class UndecoratedOptions: +class UndecoratedOptions(Configuration): level: int @@ -815,7 +807,7 @@ def level(self) -> int: @dataclass(frozen=True) -class ClosedOptions: +class ClosedOptions(Configuration): inner: CodecEntity From 848d7006db08c5c0bc08df550c24ed6da934fcc2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 21:27:14 +0200 Subject: [PATCH 092/107] refactor(zarr-metadata): `Configured`, the half of an entity that has a configuration An entity whose metadata carries a configuration adds `Configured` beside its kind -- `class GzipCodec(BytesBytesCodec, Configured)` -- which declares the `configuration` field as a `Configuration` and is what the base branches on: the plan, the constructor's check, `coerce` and `to_json` ask `issubclass(cls, Configured)` rather than whether a field happens to be there, and read the record as a typed attribute rather than through `getattr`. `with_configuration` lives on it, since only such an entity has members to replace. The entity narrows the field to its own record, which pyright accepts for a frozen dataclass. Registration refuses a `configuration` declared without the marker. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/v3/_entity.py | 104 ++++++++++-------- .../v3/chunk_grid/rectilinear.py | 3 +- .../zarr_metadata/v3/chunk_grid/regular.py | 3 +- .../v3/chunk_key_encoding/default.py | 3 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 3 +- .../src/zarr_metadata/v3/codec/blosc.py | 3 +- .../src/zarr_metadata/v3/codec/bytes.py | 3 +- .../src/zarr_metadata/v3/codec/cast_value.py | 3 +- .../src/zarr_metadata/v3/codec/gzip.py | 3 +- .../zarr_metadata/v3/codec/scale_offset.py | 3 +- .../v3/codec/sharding_indexed.py | 3 +- .../src/zarr_metadata/v3/codec/transpose.py | 3 +- .../src/zarr_metadata/v3/codec/zstd.py | 3 +- .../zarr_metadata/v3/data_type/_families.py | 3 +- .../src/zarr_metadata/v3/data_type/struct.py | 3 +- .../src/zarr_metadata/v3/entity.py | 20 ++-- .../zarr-metadata/tests/test_public_api.py | 1 + .../tests/v3/test_acme_affine.py | 3 +- .../tests/v3/test_acme_decimal.py | 3 +- .../zarr-metadata/tests/v3/test_entities.py | 3 +- .../tests/v3/test_extension_api.py | 46 +++++--- 21 files changed, 142 insertions(+), 80 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 7ed7afde4d..e70ef39254 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -27,10 +27,9 @@ from __future__ import annotations import functools -import operator from abc import ABC, abstractmethod from collections.abc import Callable, Mapping -from dataclasses import dataclass, is_dataclass, replace +from dataclasses import dataclass, replace from typing import ( TYPE_CHECKING, ClassVar, @@ -219,31 +218,39 @@ def canonical(self) -> Self: def unreadable(cls: type[MetadataEntity]) -> str | None: """Why `coerce` could not read an instance of `cls`; None if it can. - An entity's fields are `configuration`, a `Configuration` record of - its members, and at most one field the envelope's name fills. Things - that type-check cleanly and then go wrong somewhere that will not - name the class: a field of any other name; a configuration that is - not a `Configuration`, or a member of it whose annotation is not a - shape JSON takes; a `__post_init__` of the entity's own, whose rules - `coerce` would never ask; and a class variable a base annotates and - nothing sets -- `identifier` for every entity, `bounds` for an - integer type -- which the first lookup would fail. Registration asks, - and refuses the class with the answer. + An entity's fields are `configuration`, which a `Configured` entity + narrows to its own `Configuration` record, and at most one field the + envelope's name fills. Things that type-check cleanly and then go + wrong somewhere that will not name the class: a field of any other + name; a `configuration` on an entity that is not `Configured`; a + configuration that is not a `Configuration`, or a member of it whose + annotation is not a shape JSON takes; a `__post_init__` of the + entity's own, whose rules `coerce` would never ask; and a class + variable a base annotates and nothing sets -- `identifier` for every + entity, `bounds` for an integer type -- which the first lookup would + fail. Registration asks, and refuses the class with the answer. """ try: hints = field_hints(cls) except NameError as unresolved: return _unresolved(cls, unresolved) for name, annotation in hints.items(): - if name == "configuration" or is_from_name(annotation): + if name == "configuration" and issubclass(cls, Configured): continue + if is_from_name(annotation): + continue + if name == "configuration": + return ( + f"{cls.__name__} declares `configuration` without `Configured`; an entity with a " + f"configuration adds it beside its kind: class {cls.__name__}(..., Configured)" + ) return ( f"{cls.__name__} declares a field {name!r}; an entity's fields are `configuration`, " "a frozen dataclass of its members, and a name it carries marked FROM_NAME -- put " f"{name!r} in the configuration record" ) - record = hints.get("configuration") - if record is not None: + if issubclass(cls, Configured): + record = hints["configuration"] if not (isinstance(record, type) and issubclass(record, Configuration)): return ( f"{cls.__name__}: configuration is annotated {record!r}; annotate it with a frozen " @@ -380,9 +387,9 @@ class _Plan: from_name: str | None """The field the envelope's name fills, for a family; None for every other entity.""" parse: Parser[_Reading] | None - """The configuration record's parser; None for an entity with no configuration.""" - write: Callable[[MetadataEntity], dict[str, JSONValue]] | None - """The entity's configuration as the JSON object it writes; None for an entity with none.""" + """The configuration record's parser; None for an entity that is not `Configured`.""" + write: Callable[[Configured], dict[str, JSONValue]] | None + """The entity's configuration as the JSON object it writes; None for one that is not `Configured`.""" requires_configuration: bool """Whether the record has a member the document must write.""" @@ -397,18 +404,17 @@ def _plan(cls: type[MetadataEntity]) -> _Plan: """ hints = field_hints(cls) from_name = next((key for key, annotation in hints.items() if is_from_name(annotation)), None) - record = hints.get("configuration") - if record is None: + if not issubclass(cls, Configured): return _Plan(from_name, None, None, False) - if not (isinstance(record, type) and is_dataclass(record)): # pragma: no cover - refused first - msg = f"{cls.__name__}: configuration is annotated {record!r}, not a record dataclass" + record = hints["configuration"] + if not (isinstance(record, type) and issubclass(record, Configuration)): # pragma: no cover + msg = f"{cls.__name__}: configuration is annotated {record!r}, not a Configuration" raise TypeError(msg) required = any(not is_optional(annotation) for annotation in field_hints(record).values()) writes: RecordWriter = record_writer(record, _nested_field_writer) - configuration_of = operator.attrgetter("configuration") - def write(entity: MetadataEntity) -> dict[str, JSONValue]: - return writes(configuration_of(entity)) + def write(entity: Configured) -> dict[str, JSONValue]: + return writes(entity.configuration) return _Plan(from_name, parser(record, _nested_field), write, required) @@ -432,6 +438,31 @@ def problems(self) -> Iterator[ValidationProblem]: yield from () +@dataclass(frozen=True) +class Configured: + """The half of an entity that has a configuration. + + An entity whose metadata carries a `configuration` object adds this + beside its kind -- `class GzipCodec(BytesBytesCodec, Configured)` -- + and narrows the field to its own record: `configuration: + GzipOptions`. What the layer does with a configuration -- parse it, + ask its rules, write it back, replace members of it -- is done here + or asked of this, and an entity that is not `Configured` has none + of it: its metadata is a bare name. + """ + + configuration: Configuration + + def with_configuration(self, **changes: object) -> Self: + """This entity with these configuration members changed. + + `codec.with_configuration(typesize=UNSET)` is the record replaced + member by member and the entity rebuilt around it, so the + constructor checks the result as it checks any other. + """ + return replace(self, configuration=replace(self.configuration, **changes)) + + @dataclass(frozen=True) class MetadataEntity(ABC): """One named entity, coerced from its metadata. @@ -483,9 +514,8 @@ def __post_init__(self) -> None: plan = _plan(type(self)) name = self.identifier if plan.from_name is None else getattr(self, plan.from_name) first = next(type(self).name_problems(name), None) - configuration = getattr(self, "configuration", None) - if first is None and isinstance(configuration, Configuration): - first = next(configuration.problems(), None) + if first is None and isinstance(self, Configured): + first = next(self.configuration.problems(), None) if first is not None: raise MetadataValidationError((first,)) @@ -522,7 +552,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: members[plan.from_name] = name reading = _Reading(context, []) own: tuple[ValidationProblem, ...] = () - if plan.parse is None: + if not issubclass(cls, Configured) or plan.parse is None: own = tuple( found for key in (given or {}) @@ -569,19 +599,6 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: return None, found return cls(**members), found - def with_configuration(self, **changes: object) -> Self: - """This entity with these configuration members changed. - - `codec.with_configuration(typesize=UNSET)` is the record replaced - member by member and the entity rebuilt around it, so the - constructor checks the result as it checks any other. - """ - current = getattr(self, "configuration", None) - if not is_dataclass(current) or isinstance(current, type): - msg = f"{type(self).__name__} has no configuration" - raise TypeError(msg) - return replace(self, configuration=replace(current, **changes)) - def canonical(self) -> Self: """This entity in the simplest form that means the same thing. @@ -622,7 +639,7 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: if plan.from_name is not None: carried = getattr(self, plan.from_name) name = carried if isinstance(carried, str) else name - if plan.write is None: + if not isinstance(self, Configured) or plan.write is None: return name configuration = plan.write(self) if len(configuration) == 0: @@ -770,6 +787,7 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "CodecEntity", "Coerced", "Configuration", + "Configured", "DataTypeEntity", "Loc", "MetadataEntity", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index b9f57a8c10..6f9187e22b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -13,6 +13,7 @@ from zarr_metadata.v3._entity import ( ChunkGridEntity, Configuration, + Configured, Loc, is_integer, problem, @@ -192,7 +193,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class RectilinearChunkGrid(ChunkGridEntity): +class RectilinearChunkGrid(ChunkGridEntity, Configured): """The `rectilinear` chunk grid, coerced from its metadata.""" configuration: RectilinearChunkGridOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index b4b6950757..08e1722659 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -13,6 +13,7 @@ from zarr_metadata.v3._entity import ( ChunkGridEntity, Configuration, + Configured, problem, ) from zarr_metadata.v3._parts import ChunkGrid @@ -78,7 +79,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class RegularChunkGrid(ChunkGridEntity): +class RegularChunkGrid(ChunkGridEntity, Configured): """The `regular` chunk grid, coerced from its metadata.""" configuration: RegularChunkGridOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 463bf8bb81..c1300e7ba2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -16,6 +16,7 @@ from zarr_metadata.v3._entity import ( ChunkKeyEncodingEntity, Configuration, + Configured, ) DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" @@ -79,7 +80,7 @@ class DefaultChunkKeyEncodingOptions(Configuration): @dataclass(frozen=True) -class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity): +class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity, Configured): """The `default` chunk key encoding, coerced from its metadata.""" configuration: DefaultChunkKeyEncodingOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index d3aedd1961..29e7b43f4d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -22,6 +22,7 @@ from zarr_metadata.v3._entity import ( ChunkKeyEncodingEntity, Configuration, + Configured, ) V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" @@ -85,7 +86,7 @@ class V2ChunkKeyEncodingOptions(Configuration): @dataclass(frozen=True) -class V2ChunkKeyEncoding(ChunkKeyEncodingEntity): +class V2ChunkKeyEncoding(ChunkKeyEncodingEntity, Configured): """The `v2` chunk key encoding, coerced from its metadata.""" configuration: V2ChunkKeyEncodingOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 3c19c10beb..8908c6a0bc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -14,6 +14,7 @@ from zarr_metadata.v3._entity import ( BytesBytesCodec, Configuration, + Configured, ) if TYPE_CHECKING: @@ -131,7 +132,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class BloscCodec(BytesBytesCodec): +class BloscCodec(BytesBytesCodec, Configured): """The `blosc` codec, coerced from its metadata. Everything blosc knows about itself: the shape its metadata takes, the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index e09c08899e..1a0d703a61 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -14,6 +14,7 @@ from zarr_metadata.v3._entity import ( ArrayBytesCodec, Configuration, + Configured, DataTypeEntity, problem, ) @@ -87,7 +88,7 @@ class BytesOptions(Configuration): @dataclass(frozen=True) -class BytesCodec(ArrayBytesCodec): +class BytesCodec(ArrayBytesCodec, Configured): """The `bytes` codec, coerced from its metadata. `endian` is optional and absent means something: a one-byte data type diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 4285ef6a27..c89cc4399a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -15,6 +15,7 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, + Configured, DataTypeEntity, Opaque, ) @@ -136,7 +137,7 @@ class CastValueOptions(Configuration): @dataclass(frozen=True) -class CastValueCodec(ArrayArrayCodec): +class CastValueCodec(ArrayArrayCodec, Configured): """The `cast_value` codec, coerced from its metadata. Holds the data type it casts to, so like `sharding_indexed` it is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 671442d017..8a77eb73c7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -13,6 +13,7 @@ from zarr_metadata.v3._entity import ( BytesBytesCodec, Configuration, + Configured, ) if TYPE_CHECKING: @@ -82,7 +83,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class GzipCodec(BytesBytesCodec): +class GzipCodec(BytesBytesCodec, Configured): """The `gzip` codec, coerced from its metadata.""" configuration: GzipOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 622967253c..cde07cd1a4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -15,6 +15,7 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, + Configured, ) from zarr_metadata.v3._parts import ArrayParts @@ -97,7 +98,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class ScaleOffsetCodec(ArrayArrayCodec): +class ScaleOffsetCodec(ArrayArrayCodec, Configured): """The `scale_offset` codec, coerced from its metadata. Both members are optional and any JSON scalar is well-typed here; what diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 1f3a486337..a52e383339 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -17,6 +17,7 @@ ArrayBytesCodec, CodecEntity, Configuration, + Configured, Opaque, problem, ) @@ -118,7 +119,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class ShardingIndexedCodec(ArrayBytesCodec): +class ShardingIndexedCodec(ArrayBytesCodec, Configured): """The `sharding_indexed` codec, coerced from its metadata. Holds two codec pipelines, so it is one of the few entities that diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 9a3ec65677..da69962a14 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -13,6 +13,7 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, + Configured, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -86,7 +87,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class TransposeCodec(ArrayArrayCodec): +class TransposeCodec(ArrayArrayCodec, Configured): """The `transpose` codec, coerced from its metadata.""" configuration: TransposeOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index c877e976ea..151b9c69ee 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -16,6 +16,7 @@ from zarr_metadata.v3._entity import ( BytesBytesCodec, Configuration, + Configured, ) if TYPE_CHECKING: @@ -93,7 +94,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class ZstdCodec(BytesBytesCodec): +class ZstdCodec(BytesBytesCodec, Configured): """The `zstd` codec, coerced from its metadata.""" configuration: ZstdOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 762e498cd0..71efa628fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -20,6 +20,7 @@ from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( Configuration, + Configured, DataTypeEntity, StorageClass, is_integer, @@ -186,7 +187,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class NumpyTimeDataType(DataTypeEntity): +class NumpyTimeDataType(DataTypeEntity, Configured): """A numpy time scalar: a signed 64-bit count of units, or `NaT`. The two time types share their configuration -- a unit and a scale diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index b6b1ddcb6c..e22bcba804 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -15,6 +15,7 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( Configuration, + Configured, DataTypeEntity, Loc, Opaque, @@ -141,7 +142,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class StructDataType(DataTypeEntity): +class StructDataType(DataTypeEntity, Configured): """The `struct` data type, coerced from its metadata. A record of named fields, each with a data type of its own -- so this diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 7dbeb8777b..69b1ea8f07 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -39,7 +39,8 @@ **Writing an extension.** Subclass the kind of thing it is -- a codec's kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity` or -`StorageTransformerEntity`; declare the configuration as a frozen +`StorageTransformerEntity`, with `Configured` beside it if the metadata +carries a configuration; declare that configuration as a frozen `Configuration` of its members, with every rule finer than a type in its `problems`, and name it in the entity's one field, `configuration`; add the class to a scope. Complete; runnable given a `document`: @@ -54,6 +55,7 @@ UNSET, BytesBytesCodec, Configuration, + Configured, ValidationProblem, ) @@ -70,7 +72,7 @@ def problems(self) -> Iterator[ValidationProblem]: ) @dataclass(frozen=True) - class AcmeLz4Codec(BytesBytesCodec): + class AcmeLz4Codec(BytesBytesCodec, Configured): configuration: AcmeLz4Options # the shape of the metadata: a name, and a configuration identifier: ClassVar[str] = "acme.lz4" @@ -80,9 +82,10 @@ class AcmeLz4Codec(BytesBytesCodec): validate_array_metadata_v3(document, context=SCOPE) An entity has the shape of its metadata: a name, which is the class, -and a configuration, which is a record dataclass named in the one field -`configuration`. The record's fields are the only place the members -are written. Which members exist, which may be left out (the type +and, for a `Configured` one, a configuration, which is a record +dataclass named in the one field `configuration`; an entity of a bare +name is not `Configured` and has no field. The record's fields are the +only place the members are written. Which members exist, which may be left out (the type admits `UNSET`), how each one is type-checked, and how each is written back are all read off the annotations, and the shapes are the ones JSON takes: `int`, `float` (any JSON number), `bool`, `str`, @@ -153,8 +156,9 @@ class AcmeLz4Codec(BytesBytesCodec): Registration is the one moment an entity is refused, with a message that says what to write: a class without `@dataclass`, a codec subclassing `CodecEntity` instead of a kind, a field other than -`configuration` and a carried name, a configuration that is not a -record dataclass, a member whose annotation is not a shape JSON takes +`configuration` and a carried name, a `configuration` without +`Configured`, a configuration that is not a `Configuration` record, a +member whose annotation is not a shape JSON takes -- a nested entity without `Opaque` among them -- a `__post_init__` of the entity's own, a class variable a base annotates and nothing sets, and what a kind leaves abstract. Everything @@ -198,6 +202,7 @@ class AcmeLz4Codec(BytesBytesCodec): CodecEntity, Coerced, Configuration, + Configured, DataTypeEntity, Loc, MetadataEntity, @@ -235,6 +240,7 @@ class AcmeLz4Codec(BytesBytesCodec): "Coerced", "ComplexDataType", "Configuration", + "Configured", "Context", "DataTypeEntity", "Extents", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 542b697960..9bed6face4 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -296,6 +296,7 @@ def test_all_is_grouped_and_unique() -> None: "Context", "Coerced", "Configuration", + "Configured", "ChunkGrid", "ArrayParts", "ArrayDocumentV3", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 76be17d5c4..3234ee43bb 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -27,6 +27,7 @@ ArrayDocumentV3, ArrayParts, Configuration, + Configured, DataTypeEntity, MetadataValidationError, Opaque, @@ -74,7 +75,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeAffineCodec(ArrayArrayCodec): +class AcmeAffineCodec(ArrayArrayCodec, Configured): """`x * scale + offset`, stored as `dtype` if one is named.""" configuration: AcmeAffineOptions diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 16c781fe1a..e860b3590c 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -20,6 +20,7 @@ from zarr_metadata.v3.entity import ( Configuration, + Configured, DataTypeEntity, Loc, MetadataValidationError, @@ -96,7 +97,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeDecimalDataType(DataTypeEntity): +class AcmeDecimalDataType(DataTypeEntity, Configured): """The `acme.decimal` data type, coerced from its metadata.""" configuration: AcmeDecimalOptions diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 94c86ccfe0..c8e3d2619e 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -83,6 +83,7 @@ ChunkKeyEncodingEntity, CodecEntity, Configuration, + Configured, DataTypeEntity, MetadataEntity, StorageTransformerEntity, @@ -743,7 +744,7 @@ class AcmeShardCacheOptions(Configuration): @dataclasses.dataclass(frozen=True) -class AcmeShardCache(StorageTransformerEntity): +class AcmeShardCache(StorageTransformerEntity, Configured): """A third-party storage transformer with a member canonical form drops.""" configuration: AcmeShardCacheOptions diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 0f9fa8716b..9e5df5c2af 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -31,6 +31,7 @@ ChunkKeyEncodingEntity, CodecEntity, Configuration, + Configured, Context, DataTypeEntity, IntegerDataType, @@ -62,7 +63,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeLz4Codec(BytesBytesCodec): +class AcmeLz4Codec(BytesBytesCodec, Configured): """A third-party compressor.""" configuration: AcmeLz4Options @@ -195,7 +196,7 @@ def test_an_absent_optional_member_is_read_as_unset_whatever_its_default() -> No # `UNSET` in the record, so no field's default decides what a # document said. @dataclass(frozen=True) - class Defaulted(BytesBytesCodec): + class Defaulted(BytesBytesCodec, Configured): configuration: DefaultedOptions identifier: ClassVar[str] = "acme.defaulted" @@ -350,7 +351,7 @@ def test_error_a_member_needs_a_check_from_somewhere() -> None: # parser, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. @dataclass(frozen=True) - class Structured(BytesBytesCodec): + class Structured(BytesBytesCodec, Configured): configuration: StructuredOptions identifier: ClassVar[str] = "acme.structured" @@ -371,7 +372,7 @@ class AcmeWrapperOptions(Configuration): @dataclass(frozen=True) -class AcmeWrapperCodec(BytesBytesCodec): +class AcmeWrapperCodec(BytesBytesCodec, Configured): """A codec that applies another codec after its own step.""" configuration: AcmeWrapperOptions @@ -458,7 +459,7 @@ def test_canonical_is_the_entity_s_own_and_reaches_what_it_contains() -> None: # that `noshuffle` ignores, and the frame of 0 that means "unframed" # is dropped -- with nothing to call `super()` for. @dataclass(frozen=True) - class AcmeFramedCodec(BytesBytesCodec): + class AcmeFramedCodec(BytesBytesCodec, Configured): configuration: AcmeFramedOptions identifier: ClassVar[str] = "acme.framed" @@ -498,7 +499,7 @@ def test_error_a_nested_field_names_a_kind() -> None: # `MetadataEntity` is of no kind, so a field typed as one could not # be resolved through any scope. @dataclass(frozen=True) - class Vague(BytesBytesCodec): + class Vague(BytesBytesCodec, Configured): configuration: VagueOptions identifier: ClassVar[str] = "acme.vague" @@ -549,7 +550,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeBlockCodec(BytesBytesCodec): +class AcmeBlockCodec(BytesBytesCodec, Configured): """A codec whose block size must be a power of two.""" configuration: AcmeBlockOptions @@ -603,7 +604,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeRangeCodec(BytesBytesCodec): +class AcmeRangeCodec(BytesBytesCodec, Configured): """A codec with two rules, so that one can fail after another.""" configuration: AcmeRangeOptions @@ -667,7 +668,7 @@ class Local(TypedDict, closed=True): depth: int @dataclass(frozen=True) - class Localized(BytesBytesCodec): + class Localized(BytesBytesCodec, Configured): configuration: LocalizedOptions identifier: ClassVar[str] = "acme.localized" @@ -718,7 +719,7 @@ def test_a_slotted_entity_is_accepted() -> None: # `@dataclass(slots=True)` builds the class twice; registration sees # the second, whose members are slot descriptors. @dataclass(frozen=True, slots=True) - class AcmeSlotted(BytesBytesCodec): + class AcmeSlotted(BytesBytesCodec, Configured): configuration: AcmeSlottedOptions identifier: ClassVar[str] = "acme.slotted" @@ -755,7 +756,7 @@ def test_a_number_member_is_a_float_field() -> None: # point and refuses a bool, which is what a document's `2` and `true` # deserve. @dataclass(frozen=True) - class AcmeScaled(ArrayArrayCodec): + class AcmeScaled(ArrayArrayCodec, Configured): configuration: AcmeScaledOptions identifier: ClassVar[str] = "acme.scaled" @@ -793,7 +794,7 @@ def test_error_an_entity_must_be_a_dataclass() -> None: # Class creation runs before `@dataclass` and cannot see it missing; # registration can, and says so instead of the first `coerce` failing # with the base class's `__init__`. - class Undecorated(BytesBytesCodec): + class Undecorated(BytesBytesCodec, Configured): configuration: UndecoratedOptions identifier: ClassVar[str] = "acme.undecorated" @@ -814,7 +815,7 @@ class ClosedOptions(Configuration): def test_error_a_nested_field_admits_opaque() -> None: # What the field holds when the inner name is out of scope. @dataclass(frozen=True) - class Closed(BytesBytesCodec): + class Closed(BytesBytesCodec, Configured): configuration: ClosedOptions identifier: ClassVar[str] = "acme.closed" @@ -871,3 +872,22 @@ def test_error_a_list_of_problem_tuples_is_refused() -> None: # the constructor and fail inside `coerce`, far from the mistake. with pytest.raises(TypeError, match="collect with `extend`, not `append`"): MetadataValidationError([problem(("a",), "bad a")]) # pyright: ignore[reportArgumentType] + + +@dataclass(frozen=True) +class UnmarkedOptions(Configuration): + level: int + + +def test_error_a_configuration_needs_configured_beside_the_kind() -> None: + # The marker is what the layer branches on: an entity that declares + # the field without it has a field of a name the layer does not read. + @dataclass(frozen=True) + class Unmarked(BytesBytesCodec): + configuration: UnmarkedOptions + + identifier: ClassVar[str] = "acme.unmarked" + variable_size: ClassVar[bool] = False + + with pytest.raises(TypeError, match="declares `configuration` without `Configured`"): + CORE_AND_EXTENSIONS.extended_with(Unmarked) From 764cdc976571bb2dfd680e4514d2aebe4f94ddcb Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 21:28:18 +0200 Subject: [PATCH 093/107] docs(zarr-metadata): the fragments describe the configuration record Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.10.md | 24 ++++----- .../zarr-metadata/changes/4379.feature.7.md | 52 ++++++++++--------- .../zarr-metadata/changes/4379.feature.md | 7 +-- packages/zarr-metadata/changes/4379.misc.1.md | 6 +-- packages/zarr-metadata/changes/4379.misc.2.md | 20 ++++++- 5 files changed, 66 insertions(+), 43 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index 78019a05fc..132447b434 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -1,17 +1,17 @@ An entity is the value guarantee, not just the type one. Construction -refuses values the spec disallows, so `BloscCodec(clevel=99)` raises -rather than serializing a document no reader will accept, and there is no -such thing as an invalid entity to hold. Code that wants unvalidated -metadata already has somewhere to put it: `Opaque`, which carries the -JSON verbatim. +refuses values the spec disallows, so `BloscCodec(BloscOptions(clevel=99))` +raises rather than serializing a document no reader will accept, and +there is no such thing as an invalid entity to hold. Code that wants +unvalidated metadata already has somewhere to put it: `Opaque`, which +carries the JSON verbatim, and the configuration record itself, which is +the options as written and not yet judged. -One piece: a function of the instance that yields the values the spec -disallows as it finds them, bound on the class as `problems`. The -constructor stops at the first and raises `MetadataValidationError`; -`coerce` runs it to the end and reports every problem in the document -instead of raising; a consumer holding an entity may run it too, and -stop or collect as it likes. `create_unchecked(**members)` builds the -record without the check, for a reader that judges afterwards. +One piece: the configuration record's own `problems`, which yields the +values the spec disallows as it finds them. The entity's constructor +stops at the first and raises `MetadataValidationError`; `coerce` asks +the record before it builds anything and reports every problem in the +document instead of raising; a reader with a record asks it too, +`BloscOptions(...).problems()`, and stops or collects as it likes. `must_understand` moves to a class variable and out of the configuration entirely. It is a property of the *kind* of metadata -- a codec is diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 2c920f1ea7..644799e65c 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -27,8 +27,12 @@ nothing about whether a shard's pipelines are well formed and silencing them would lose a real judgment. A *required* one stops the entity, because there is no honest reading of a `blosc` whose level is a string. -The type judgment is read off the entity's own fields rather than -written twice. Which members exist, which may be left out (the type +An entity has the shape of its metadata: its name is the class, and its +configuration, for an entity that adds `Configured` beside its kind, is +a frozen `Configuration` record named in the one field `configuration` +-- `class GzipCodec(BytesBytesCodec, Configured)` with `configuration: +GzipOptions`. The type judgment is read off the record's fields rather +than written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field annotations: `int`, `float` for any number, `bool`, `str`, a `Literal` of names, the JSON-value alias, an array homogeneous or fixed, a union @@ -37,17 +41,17 @@ dataclass, an object of undeclared keys as `Mapping[str, V]`, a `NewType` as the type it names, and a nested metadata field, written as its kind with `Opaque`. That covers every member this package models; an annotation outside them is refused -at registration, and `Annotated[str, FROM_NAME]` marks the one field -carried by the envelope's name rather than a configuration key (`r`), -which `coerce` fills from the envelope. Whether a configuration is required -follows too, since the spec ties the bare-name spelling to whether any member is -required. +at registration, and `Annotated[str, FROM_NAME]` marks the one field of +the entity itself, carried by the envelope's name rather than a +configuration key (`r`), which `coerce` fills from the envelope. +Whether a configuration is required follows too, since the spec ties the +bare-name spelling to whether any member is required. -Nothing is derived from the fields ahead of time: `coerce` parses a -configuration against them as it reads it. Field annotations are -resolved per class, skipping class variables by text, so a `ClassVar` -naming something imported only for the type checker cannot fail -registration. +The record's parser and writer are compiled once per class from its +annotations, and `coerce` runs the parser over the configuration as it +reads it. Field annotations are resolved per class, skipping class +variables by text, so a `ClassVar` naming something imported only for +the type checker cannot fail registration. One diagnostic became more precise: a malformed `[value, count]` pair in a rectilinear grid is now reported at the offending element inside the @@ -57,10 +61,10 @@ An entity that contains other entities declares the field -- `data_type: DataTypeEntity | Opaque`, `codecs: tuple[CodecEntity | Opaque, ...]`, a record holding one -- and the contained entity is read through the scope the containing one is read in, at whatever depth the -annotation puts it. Writing it back and putting it in canonical form -are the containing entity's own two lines, `written(self.inner)` in -`to_json` and `replace(self, inner=canonicalized(self.inner))` in -`canonical`, the same way it writes and simplifies its own members. +annotation puts it. Writing it back is the base's, through the contained +entity's own `to_json`; putting it in canonical form is the containing +entity's one line, `self.with_configuration(inner=self.inner.canonical())` +in `canonical`, the same way it simplifies its own members. `canonical` is the entity itself by default, overridden where two spellings of its members mean the same -- a rectilinear dimension's run-length encoding, a `typesize` that `noshuffle` ignores. A field @@ -69,19 +73,19 @@ refused at registration: no scope could place the one, and the other lies about what the field holds when the inner name is out of scope. Everything finer than a type -- a bound, a rule about one member, a -rule that reads two members together -- is a function of the entity's -instance, in plain code, that yields each problem as it finds it, -located relative to the configuration, and is bound on the class as -`problems`. The constructor stops at the first, so `BloscCodec(clevel=99)` -raises `MetadataValidationError`; `coerce` runs it to the end and -reports every problem in the document; a consumer holding an entity may -run it too, stopping or collecting as it needs. The space of +rule that reads two members together -- is the record's own `problems`, +in plain code, yielding each problem as it finds it, located relative +to the configuration. The entity's constructor stops at the first, so +`BloscCodec(BloscOptions(clevel=99))` raises `MetadataValidationError`; +`coerce` asks the record before it builds anything and reports every +problem in the document; a reader with a record asks it too, stopping +or collecting as it needs. The space of refinements is too wide to capture statically, and reading them in one place per entity is what will show where a shared form is worth having. The set of annotation shapes the parser reads is closed: the shapes JSON takes, and no others. A field annotation outside them is refused at registration, and the field is written as one of them instead, with -any finer rule in the function bound as `problems`. The parser is one module that knows +any finer rule in the record's `problems`. The parser is one module that knows nothing of entities; the entity layer hands it the one shape of its own, a field holding an entity, as a leaf it reads at any depth. diff --git a/packages/zarr-metadata/changes/4379.feature.md b/packages/zarr-metadata/changes/4379.feature.md index 1da2ed4dbb..fadf1ee2c1 100644 --- a/packages/zarr-metadata/changes/4379.feature.md +++ b/packages/zarr-metadata/changes/4379.feature.md @@ -9,9 +9,10 @@ data type, chunk grid and chunk key encoding — is a class that answers for itself: - `coerce` reads raw metadata into the entity, or says why it is not one; -- `problems`, a function of the instance bound on the class, yields the - values the spec disallows as it finds them; the constructor stops at - the first and raises, and `coerce` reports every one instead; +- its configuration is a record of its own, `GzipOptions`, whose + `problems` yields the values the spec disallows as it finds them; the + entity's constructor stops at the first and raises, and `coerce` + reports every one instead; - `to_json` writes it back as it was read, and `canonical` gives its simplest equivalent spelling; - `incoming_problems`, `shape_problems`, `fill_value_problems` and diff --git a/packages/zarr-metadata/changes/4379.misc.1.md b/packages/zarr-metadata/changes/4379.misc.1.md index 2887f26117..7d7c6c34c0 100644 --- a/packages/zarr-metadata/changes/4379.misc.1.md +++ b/packages/zarr-metadata/changes/4379.misc.1.md @@ -12,9 +12,9 @@ TypedDicts. The tables cannot drift now, because there is one of each fact instead of four. What replaces the drift tests is a correspondence test: an entity's -dataclass fields, its configuration TypedDict, and its member table are -three spellings of one set, and whether the bare-name spelling is -allowed follows from the TypedDict's required keys. +configuration record and its configuration TypedDict are two spellings +of one set of members, with the same requiredness, and whether the +bare-name spelling is allowed follows from that. One more table went the same way: `zarr_metadata.v3.codec.kind`, which sorted codec *names* into the spec's three pipeline kinds. A name does diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 97fef360ab..10746b5111 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -96,7 +96,7 @@ registration checks for that are gone. Which of a document's fields holds which kind is the document's own knowledge, in one place; the entity layer no longer names a document field. -`to_json` is written once, in the base, from the fields: `writer_for` +`to_json` is written once, in the base, from the configuration record: `writer_for` is the parser's inverse over the same annotation, so what `coerce` reads from a document, `to_json` puts back -- the bare name when every member is absent, the object otherwise, a contained entity through its own @@ -134,3 +134,21 @@ compiled into it, and the plan for a class, like its resolved field annotations, is a pure function of the class and cached as one. A document with a shard, six codecs and a nested pipeline reads in 85 µs where it took 199, and one codec in 6 µs where it took 21. + +An entity has the shape of its metadata. Its name is the class, and its +configuration, for an entity that adds `Configured` beside its kind, is +a frozen `Configuration` record named in the one field `configuration` +-- `class GzipCodec(BytesBytesCodec, Configured)` with `configuration: +GzipOptions`, read as `codec.configuration.level` or through a +`@property` the entity adds for a member it wants at the top level. The +record's fields are the members and its `problems` the rules on them, +so the lift of configuration keys to the entity, which nothing in Python +expressed, is gone with the per-member loop in `coerce` that did it: a +configuration is parsed and written by the parser's and the writer's +record shapes. `with_configuration(**changes)` is the entity with members +of its configuration replaced, checked as any construction is. A record +is the options as written, judged on request; the entity is the value +guarantee, and `coerce` asks the record before it builds one, so no +unchecked construction path exists. The rule a family has about its +name -- `r` a multiple of 8 -- is the entity's `name_problems`, since +a name is not configuration. From 917a641320fe2a7613083df81403a576eb339889 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 21 Sep 2026 23:06:37 +0200 Subject: [PATCH 094/107] fix(zarr-metadata): what the adversarial review of the entity layer found Four reviewers (correctness, extension author, simplicity, zarrs and TensorStore parity) read 764cdc976. The corrections that need no decision: - A pipeline the document did not write as an array was read as an empty one and judged "no array->bytes codec" beside the structural problem. Nothing was read, so nothing is judged. On the 40k-document corpus that is the whole difference: 4,262 occurrences of that one message lost, every one on a document whose `codecs` is absent or not an array and which keeps its structural problem there; 0 verdict flips, 0 problems gained. - A name in scope but of another kind -- `transpose` where a `BytesBytesCodec` is asked for -- came back `out_of_scope`, for a reader to resolve elsewhere. It is `invalid`, with a problem that names the kind it is. - The checker: a `Literal` of booleans had the shape of integers, and a `Literal` mixing shapes had one it did not, so a union could send a value to the wrong branch; describing a mixed `Literal` crashed in `sorted`; `is_class_var` missed `t.ClassVar` under PEP 649; `field_hints` handed out its cached dict; a fixed tuple of the wrong length raised `zip`'s error rather than the JSON one. - Registration refuses two more things, with a message: a member that is itself a `Configuration`, whose rules nothing would ask, and a `__post_init__` on a record, which would stop at the first problem where `coerce` reports every one. A `NameError` inside a nested record gets the message an entity's gets. - `Extents` was a string alias, so `Extents | None` raised in an extension's annotation. - The `*Options` records are exported from their modules, since hand construction needs them; the public-name grammar learns the role. - Prose. The door's example is complete and runs as written, and says what `transition` may return, what `storage_class()` answers, how a family is written, and that only `| UNSET` makes a member optional to a document. The boundary paragraph of 4379.feature.7 described a fallback the branch does not have: an ill-typed optional member stops the entity, as the tests say, and what it contains is still read. `canonicalize_array_metadata_v3` says that a canonical form breaking its own rules raises, since the document was valid. Stale names (`blosc_problems`, `raw_bytes_problems`, `canonicalized`, `__post_init__` as the place for rules) are updated, and a duplicate import in `struct` is gone. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 12 +-- .../src/zarr_metadata/model/_validation.py | 7 +- .../src/zarr_metadata/rules/_documents.py | 9 ++ .../src/zarr_metadata/v3/_document.py | 20 ++++- .../src/zarr_metadata/v3/_entity.py | 31 +++++-- .../src/zarr_metadata/v3/_parts.py | 2 +- .../src/zarr_metadata/v3/_registry.py | 37 ++++++-- .../src/zarr_metadata/v3/_typed_json.py | 50 ++++++----- .../v3/chunk_grid/rectilinear.py | 1 + .../zarr_metadata/v3/chunk_grid/regular.py | 1 + .../v3/chunk_key_encoding/default.py | 1 + .../zarr_metadata/v3/chunk_key_encoding/v2.py | 1 + .../src/zarr_metadata/v3/codec/blosc.py | 3 +- .../src/zarr_metadata/v3/codec/bytes.py | 1 + .../src/zarr_metadata/v3/codec/cast_value.py | 1 + .../src/zarr_metadata/v3/codec/gzip.py | 1 + .../zarr_metadata/v3/codec/scale_offset.py | 1 + .../v3/codec/sharding_indexed.py | 1 + .../src/zarr_metadata/v3/codec/transpose.py | 1 + .../src/zarr_metadata/v3/codec/zstd.py | 1 + .../zarr_metadata/v3/data_type/_families.py | 1 + .../src/zarr_metadata/v3/data_type/raw.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 3 +- .../src/zarr_metadata/v3/entity.py | 80 ++++++++++++------ .../tests/rules/test_v3_array_rules.py | 7 ++ .../zarr-metadata/tests/test_public_api.py | 4 + .../zarr-metadata/tests/v3/test_entities.py | 80 +++++++++++++++++- .../tests/v3/test_extension_api.py | 84 ++++++++++++++++++- 28 files changed, 364 insertions(+), 79 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 644799e65c..8abbfc26db 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -21,11 +21,13 @@ are where the widths live: every integer type differs from every other only in its bounds, every float only in its hex parser, every complex only in its component type. -The boundary is worth recording: an *optional* member that fails its -type check falls back to absent, because a bad `index_location` says -nothing about whether a shard's pipelines are well formed and silencing -them would lose a real judgment. A *required* one stops the entity, -because there is no honest reading of a `blosc` whose level is a string. +The boundary is worth recording: a member that fails its type check +stops the entity, optional or not. There is no honest reading of a +`blosc` whose level is a string, and building one around the hole would +have its rules see the member as absent and add a second, contradictory +problem at the same location. What the entity contains is still read -- +a shard's inner codecs are judged whatever its `index_location` says -- +and only the judgments the shard itself would make wait on the fix. An entity has the shape of its metadata: its name is the class, and its configuration, for an entity that adds `Configured` beside its kind, is diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 5c77338672..58b52158b4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -57,8 +57,11 @@ class ValidationProblem: """A single structural problem found while validating a metadata document. - `loc` is the path from the document root to the offending value, e.g. - `("codecs", 0, "name")`. An empty `loc` refers to the document as a whole. + `loc` is the path from the root of what was judged to the offending + value, e.g. `("codecs", 0, "name")`, and an empty `loc` refers to that + root. The root is the document for the validators, the one field for + a scope's `coerce`, and the configuration for an entity's rules and + its constructor. `kind` classifies the failure mode for programmatic dispatch; `message` is the human-readable description. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index 61a60709e0..e6e23d07d2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -242,6 +242,15 @@ def canonicalize_array_metadata_v3( problems come back with the semantic ones, and the result is `Invalid` rather than a canonical document. The document is read once: the entities that judge it are the entities that are rewritten. + + Tell the two apart with `result.valid is True` or + `isinstance(result, Invalid)`; pyright narrows the literal on a + comparison, not on `if result.valid`. + + An entity whose canonical form breaks its own rules raises + `MetadataValidationError` from here, as its constructor does: that + is a bug in the entity, not a verdict on the document, which was + valid. """ normalized = arrays_to_tuples(document) problems = _validate_structure_v3(normalized) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 7a1ac22e96..78e37f3ecf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -77,11 +77,17 @@ def problems(self) -> tuple[ValidationProblem, ...]: """ # No per-entity value problems: an entity exists only if its own # values are allowed, so `read_array_v3` has already reported any. + # A pipeline the document did not write as an array was not read, + # and an empty one would be judged for a verdict about nothing. return ( *_fill_value_problems(self), *_grid_problems(self), *_dimension_names_problems(self), - *chain_problems(self.codecs, self.parts, ("codecs",)), + *( + chain_problems(self.codecs, self.parts, ("codecs",)) + if _listed(self.document, "codecs") is not None + else () + ), ) def canonical(self) -> ArrayDocumentV3: @@ -174,6 +180,12 @@ def parts(self) -> ArrayParts: ) +def _listed(document: Mapping[str, object], key: str) -> Sequence[object] | None: + """What the document lists at `key`; None if it wrote no array there.""" + entries = document.get(key) + return cast("Sequence[object]", entries) if isinstance(entries, (list, tuple)) else None + + def _read_one( context: Context, kind: type[_EntityT], document: Mapping[str, object], key: str ) -> tuple[_EntityT | Opaque, tuple[ValidationProblem, ...]]: @@ -188,12 +200,12 @@ def _read_each( context: Context, kind: type[_EntityT], document: Mapping[str, object], key: str ) -> tuple[tuple[_EntityT | Opaque, ...], tuple[ValidationProblem, ...]]: """The entities of `kind` the document lists at `key`, in order.""" - entries = document.get(key) - if not isinstance(entries, (list, tuple)): + entries = _listed(document, key) + if entries is None: return (), () read: list[_EntityT | Opaque] = [] problems: list[ValidationProblem] = [] - for index, entry in enumerate(cast("Sequence[object]", entries)): + for index, entry in enumerate(entries): entity, found = context.coerce(kind, entry, (key, index), envelope_judged=True) read.append(entity) problems.extend(found) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index e70ef39254..9024a2be00 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -59,6 +59,7 @@ problem, record_writer, strip_annotation, + without_unset, ) if TYPE_CHECKING: @@ -224,8 +225,9 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: wrong somewhere that will not name the class: a field of any other name; a `configuration` on an entity that is not `Configured`; a configuration that is not a `Configuration`, or a member of it whose - annotation is not a shape JSON takes; a `__post_init__` of the - entity's own, whose rules `coerce` would never ask; and a class + annotation is not a shape JSON takes, or is itself a `Configuration`, + whose rules nothing would ask; a `__post_init__` of the entity's or + its record's own, since the rules go in `problems`; and a class variable a base annotates and nothing sets -- `identifier` for every entity, `bounds` for an integer type -- which the first lookup would fail. Registration asks, and refuses the class with the answer. @@ -260,12 +262,27 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: members = field_hints(record) except NameError as unresolved: return _unresolved(record, unresolved) + if "__post_init__" in vars(record): + return ( + f"{record.__name__} defines __post_init__; write its rules as `problems`, " + "yielding each: the entity's constructor stops at the first, `coerce` reports " + "every one" + ) unread: list[str] = [] for name, annotation in members.items(): + inner = without_unset(strip_annotation(annotation)[0]) + if isinstance(inner, type) and issubclass(inner, Configuration): + return ( + f"{cls.__name__}: {name} is annotated {inner.__name__}, a Configuration, whose " + "rules nothing would ask; a member that is an object is a plain record " + "dataclass or a TypedDict, and its rules belong to the entity's configuration" + ) try: accepted = parser_for(annotation, _nested_field) is not None except TypeError as refused: return f"{cls.__name__}: {name} {refused}" + except NameError as unresolved: + return _unresolved(record, unresolved) if not accepted: unread.append(name) if len(unread) != 0: @@ -276,7 +293,7 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " "record, Mapping[str, V], a NewType, or an entity kind with Opaque " "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in the function bound as `problems`" + "rule in the record's `problems`" ) if "__post_init__" in vars(cls): return ( @@ -470,7 +487,7 @@ class MetadataEntity(ABC): Subclasses add their configuration members as fields, which is what makes them well-typed when read: `coerce` builds one only from metadata it accepted. Built by hand, the types are the caller's - promise -- `__post_init__` judges values, not types. An optional member is + promise -- the record's `problems` judges values, not types. An optional member is typed `| UNSET` with a default of `UNSET`, so absence is representable -- and distinct from a `null` the document wrote -- and a canonical spelling can leave it out. @@ -612,7 +629,7 @@ def canonical(self) -> Self: of the entity's members mean the same -- a rectilinear dimension's run-length encoding, a `typesize` that `noshuffle` ignores -- and, in an entity that contains entities, to put those - in canonical form: `replace(self, inner=canonicalized(self.inner))`. + in canonical form: `self.with_configuration(inner=self.inner.canonical())`. """ return self @@ -629,7 +646,9 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: if you want the simplest equivalent spelling. The envelope's spelling is the one thing not preserved, because the entity does not model it: a bare name, `{"name": x}` and `{"name": x, - "configuration": {}}` all read to the same entity. + "configuration": {}}` all read to the same entity, and a + `must_understand` the document wrote is not written back, since + absent means the same as `true` and `false` is refused. An entity whose JSON is not its fields overrides this; none in the package does. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py index 10f8dcf2b1..0223c3119f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_parts.py @@ -50,7 +50,7 @@ from zarr_metadata.v3._entity import DataTypeEntity -Extents: TypeAlias = "tuple[frozenset[int] | None, ...]" +Extents: TypeAlias = tuple[frozenset[int] | None, ...] """One entry per dimension: the lengths that dimension's chunks take. A singleton is a uniform axis. `None` is an axis whose lengths this diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 7baaaa2c02..1118711c4d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -141,14 +141,18 @@ def resolve(self, kind: type[_EntityT], name: str) -> type[_EntityT] | None: a table keyed by name could not hold it; the identifier keys exist for `extended_with` to take a name over, not for lookup. """ + entity = self._claimant(kind, name) + if entity is None or not issubclass(entity, kind): + return None + return entity + + def _claimant(self, kind: type[MetadataEntity], name: str) -> type[MetadataEntity] | None: + """The entity registered under `kind`'s kind that claims `name`, whatever its subclass.""" registered = kind_of(kind) if registered is None: return None table = self.tables.get(registered, {}) - entity = next((candidate for candidate in table.values() if candidate.accepts(name)), None) - if entity is None or not issubclass(entity, kind): - return None - return entity + return next((candidate for candidate in table.values() if candidate.accepts(name)), None) def coerce( self, @@ -192,9 +196,21 @@ def coerce( name, _, malformed = named_configuration(value) if name is None or len(malformed) != 0: return Opaque(value, "invalid"), problems - entity_type = self.resolve(kind, name) + entity_type = self._claimant(kind, name) if entity_type is None: return Opaque(value, "out_of_scope"), problems + if not issubclass(entity_type, kind): + # In scope, so not for another reader to resolve: the name + # is an entity of the wrong kind for this position. + return Opaque(value, "invalid"), ( + *problems, + ValidationProblem( + loc, + f"expected {_an(kind.__name__)}, got {name!r}, " + f"{_an(_refinement(entity_type, kind).__name__)}", + "invalid_value", + ), + ) entity, found = entity_type.coerce(value, self) problems = ( *problems, @@ -205,6 +221,17 @@ def coerce( return entity, problems +def _refinement(entity: type[MetadataEntity], kind: type[MetadataEntity]) -> type[MetadataEntity]: + """The class just below `kind`'s kind that `entity` is: `ArrayArrayCodec` for a transpose codec.""" + mro = entity.__mro__ + return mro[mro.index(kind_of(kind) or MetadataEntity) - 1] + + +def _an(noun: str) -> str: + """`noun` with its indefinite article: `an ArrayArrayCodec`, `a BytesBytesCodec`.""" + return f"an {noun}" if noun[:1].upper() in "AEIOU" else f"a {noun}" + + def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: """The kind `entity` is registered under; `TypeError` for a class no scope can use. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index 2f3e76abe9..78313fe8eb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -51,7 +51,7 @@ from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem, is_json +from zarr_metadata.model._validation import ValidationProblem, is_json if TYPE_CHECKING: from zarr_metadata.model._validation import ProblemKind @@ -215,13 +215,13 @@ def is_class_var(annotation: object) -> bool: that is not available while the class is still being built. """ if isinstance(annotation, str): - stripped = annotation.strip() - return stripped.startswith(("ClassVar[", "ClassVar", "typing.ClassVar")) + head = annotation.strip().split("[", 1)[0].strip() + return head.rsplit(".", 1)[-1] == "ClassVar" return annotation is ClassVar or get_origin(annotation) is ClassVar @functools.cache -def field_hints(cls: type) -> dict[str, object]: +def field_hints(cls: type) -> Mapping[str, object]: """The dataclass fields of `cls`, resolved, base first. Each class's own annotations are resolved in that class's module, @@ -232,7 +232,8 @@ def field_hints(cls: type) -> dict[str, object]: Cached per class: a class's annotations are fixed once it exists, and resolving them costs a third of a read. A name that does not - resolve raises, and a raise is not cached. + resolve raises, and a raise is not cached. Read-only, since every + caller shares the one mapping. """ hints: dict[str, object] = {} for ancestor in reversed(cls.__mro__): @@ -245,7 +246,7 @@ def field_hints(cls: type) -> dict[str, object]: continue shell = type("_Fields", (), {"__annotations__": raw, "__module__": ancestor.__module__}) hints.update(get_type_hints(shell, include_extras=True)) - return hints + return types.MappingProxyType(hints) def declared_class_vars(cls: type) -> dict[str, type]: @@ -281,7 +282,7 @@ def describe(annotation: object) -> str: return "a JSON value" origin = get_origin(inner) if origin is Literal: - return f"one of {tuple(sorted(get_args(inner)))!r}" + return f"one of {tuple(sorted(get_args(inner), key=repr))!r}" if is_union(inner): return " or ".join(describe(branch) for branch in get_args(inner)) if origin is tuple: @@ -317,8 +318,10 @@ def shape_of(annotation: object) -> str | None: return "str" origin = get_origin(inner) if origin is Literal: - values = get_args(inner) - return "int" if all(isinstance(value, int) for value in values) else "str" + # `True` is a bool before it is an int, as `is_integer` says. + values: tuple[object, ...] = get_args(inner) + shapes = {shape_of(type(value)) for value in values} + return shapes.pop() if len(shapes) == 1 else None if origin is tuple: return "tuple" if origin in (Mapping, dict): @@ -503,9 +506,8 @@ def record_of(record: Callable[..., object], members: Members[S]) -> Parser[S]: """A member that is itself an object with declared keys, built as a dataclass. Built only from an object whose every key read; otherwise the value - comes back as it came, with the reasons. A record that refuses its - own values -- a `__post_init__` raising `MetadataValidationError` -- - is reported the same way, located under the object. + comes back as it came, with the reasons. A record is plain data: it + has no rules of its own, so building it cannot fail. """ def parse(value: object, loc: Loc, state: S) -> Parsed: @@ -515,16 +517,7 @@ def parse(value: object, loc: Loc, state: S) -> Parsed: parsed, found = _keys(members, entries, loc, state) if any(entry.kind != "unknown_key" for entry in found): return entries, found - try: - return record(**parsed), found - except MetadataValidationError as refused: - return entries, ( - *found, - *( - ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) - for entry in refused.problems - ), - ) + return record(**parsed), found return parse @@ -608,7 +601,7 @@ def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: used as it is. Closed: an annotation outside these implies no parser, and an entity declaring one is refused at registration. The field is written as one of these shapes instead, with any finer - rule in `__post_init__`. + rule in the configuration's `problems`. """ inner = without_unset(strip_annotation(annotation)[0]) found = leaf(inner) @@ -648,6 +641,15 @@ def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: # Last, because `is_dataclass` narrows what pyright knows of `inner` # for every line after it. if isinstance(inner, type) and is_dataclass(inner): + if "__post_init__" in vars(inner): + # A record is plain data, built whenever its keys read; a + # rule about it belongs in the configuration's `problems`, + # which is asked for every problem rather than the first. + msg = ( + f"{inner.__name__} defines __post_init__; a record is plain data, and a rule " + "about it belongs in the configuration's `problems`" + ) + raise TypeError(msg) members = _members_of(field_hints(inner), leaf) return None if members is None else record_of(inner, members) return None @@ -708,6 +710,8 @@ def write(value: object) -> JSONValue: if not isinstance(value, (list, tuple)): raise _not_json(value) entries = cast("list[object] | tuple[object, ...]", value) + if len(entries) != len(elements): + raise _not_json(entries) return tuple(element(entry) for element, entry in zip(elements, entries, strict=True)) return write diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 6f9187e22b..ad5b8fad60 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -122,6 +122,7 @@ def canonical_chunk_shapes( "RectilinearChunkGridMetadata", "RectilinearChunkGridName", "RectilinearChunkGridObject", + "RectilinearChunkGridOptions", "RectilinearDimSpec", "canonical_chunk_shapes", "canonical_dim_spec", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 08e1722659..55261e98b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -59,6 +59,7 @@ class RegularChunkGridObject(TypedDict, closed=True): "RegularChunkGridMetadata", "RegularChunkGridName", "RegularChunkGridObject", + "RegularChunkGridOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index c1300e7ba2..e6a63d051b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -68,6 +68,7 @@ class DefaultChunkKeyEncodingObject(TypedDict, closed=True): "DefaultChunkKeyEncodingMetadata", "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingObject", + "DefaultChunkKeyEncodingOptions", "DefaultChunkKeyEncodingSeparator", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index 29e7b43f4d..c8512cbf8a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -74,6 +74,7 @@ class V2ChunkKeyEncodingObject(TypedDict, closed=True): "V2ChunkKeyEncodingMetadata", "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingObject", + "V2ChunkKeyEncodingOptions", "V2ChunkKeyEncodingSeparator", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 8908c6a0bc..522b675728 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -87,6 +87,7 @@ class BloscCodecObject(TypedDict, closed=True): "BloscCodecMetadata", "BloscCodecName", "BloscCodecObject", + "BloscOptions", "BloscShuffle", ] @@ -146,7 +147,7 @@ class BloscCodec(BytesBytesCodec, Configured): variable_size: ClassVar[bool] = True # Every member is required but `typesize`, which only means something - # when shuffling; `blosc_problems` is where that conditional lives. + # when shuffling; `BloscOptions.problems` is where that conditional lives. @property def cname(self) -> BloscCName: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 1a0d703a61..fe0d2f0f13 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -76,6 +76,7 @@ class BytesCodecObject(TypedDict, closed=True): "BytesCodecMetadata", "BytesCodecName", "BytesCodecObject", + "BytesOptions", "Endianness", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index c89cc4399a..a4235911e5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -117,6 +117,7 @@ class CastValueCodecObject(TypedDict, closed=True): "CastValueCodecMetadata", "CastValueCodecName", "CastValueCodecObject", + "CastValueOptions", "ScalarMap", "ScalarMapEntry", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 8a77eb73c7..05190ef6b5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -66,6 +66,7 @@ class GzipCodecObject(TypedDict, closed=True): "GzipCodecMetadata", "GzipCodecName", "GzipCodecObject", + "GzipOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index cde07cd1a4..0c576f2e6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -73,6 +73,7 @@ class ScaleOffsetCodecObject(TypedDict, closed=True): "ScaleOffsetCodecMetadata", "ScaleOffsetCodecName", "ScaleOffsetCodecObject", + "ScaleOffsetOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a52e383339..c5f06fe6b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -96,6 +96,7 @@ class ShardingIndexedCodecObject(TypedDict, closed=True): "ShardingIndexedCodecMetadata", "ShardingIndexedCodecName", "ShardingIndexedCodecObject", + "ShardingIndexedOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index da69962a14..9a5918e5a0 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -63,6 +63,7 @@ class TransposeCodecObject(TypedDict, closed=True): "TransposeCodecMetadata", "TransposeCodecName", "TransposeCodecObject", + "TransposeOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index 151b9c69ee..a938de790c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -74,6 +74,7 @@ class ZstdCodecObject(TypedDict, closed=True): "ZstdCodecMetadata", "ZstdCodecName", "ZstdCodecObject", + "ZstdOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 71efa628fc..a91b91ee4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -227,6 +227,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP "FloatDataType", "IntegerDataType", "NumpyTimeDataType", + "NumpyTimeOptions", "NumpyTimeUnit", "as_sequence", "byte_values", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 328c4eee84..c39e24bc3f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -130,7 +130,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP """One byte value per byte of the scalar. A malformed name says nothing about how wide the scalar is, so - there is no length to check against; `raw_bytes_problems` reports the name. + there is no length to check against; `name_problems` reports the name. """ try: raw_bytes_dtype_name(self.data_type_name) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index e22bcba804..22dd9d9691 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -4,7 +4,7 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ -from collections.abc import Iterator, Mapping +from collections.abc import Mapping from dataclasses import dataclass, replace from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self, cast @@ -85,6 +85,7 @@ class Struct(TypedDict, closed=True): "StructField", "StructFieldComponent", "StructFillValue", + "StructOptions", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 69b1ea8f07..d2dfe8e9db 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -26,12 +26,14 @@ **What comes back.** Problems, not exceptions, wherever a document is being judged rather than demanded. `zarr_metadata.rules.validate_array_metadata_v3(document, context=...)` -returns a tuple of `ValidationProblem(loc, message, kind)`, `kind` one of -`ProblemKind`, each `loc` indexing into the document: -`("codecs", 1, "configuration", "level")`. `SCOPE.coerce(CodecEntity, entry)` -reads one metadata field as an entity of that kind and returns -`(entity, problems)` where `entity` is the entity or an `Opaque` -- never -`None` -- with `loc` relative to the entry: `("configuration", "level")`. An entity's own +returns a tuple of `ValidationProblem(loc, message, kind)`, each `loc` +indexing into the document: +`("codecs", 1, "configuration", "level")`, and `kind` one of +`invalid_type`, `invalid_value`, `missing_key`, `unknown_key` and +`invalid_json`. `SCOPE.coerce(CodecEntity, entry)` reads one metadata +field as an entity of that kind and returns `(entity, problems)` where +`entity` is the entity or an `Opaque` -- never `None` -- with `loc` +relative to the entry: `("configuration", "level")`. An entity's own `coerce(value, context)` returns `(entity or None, problems)`; that is `Coerced`. Constructing an entity by hand raises `MetadataValidationError` with `loc` relative to the configuration: `("level",)`. @@ -43,7 +45,7 @@ carries a configuration; declare that configuration as a frozen `Configuration` of its members, with every rule finer than a type in its `problems`, and name it in the entity's one field, `configuration`; -add the class to a scope. Complete; runnable given a `document`: +add the class to a scope. Complete, and runnable as written: from collections.abc import Iterator from dataclasses import dataclass @@ -79,16 +81,24 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): variable_size: ClassVar[bool] = True # a compressor: its output length is not fixed SCOPE = CORE_AND_EXTENSIONS.extended_with(AcmeLz4Codec) - validate_array_metadata_v3(document, context=SCOPE) + document = { + "zarr_format": 3, "node_type": "array", "shape": [8], "data_type": "uint8", + "fill_value": 0, "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [8]}}, + "chunk_key_encoding": "default", + "codecs": ["bytes", {"name": "acme.lz4", "configuration": {"acceleration": 3}}], + } + assert validate_array_metadata_v3(document, context=SCOPE) == () An entity has the shape of its metadata: a name, which is the class, and, for a `Configured` one, a configuration, which is a record dataclass named in the one field `configuration`; an entity of a bare name is not `Configured` and has no field. The record's fields are the -only place the members are written. Which members exist, which may be left out (the type -admits `UNSET`), how each one is type-checked, and how each is written -back are all read off the annotations, and the shapes are the -ones JSON takes: `int`, `float` (any JSON number), `bool`, `str`, +one place the entity's members are declared; the public `*Configuration` +TypedDict beside it declares the JSON, and a test holds the two to the +same keys. Which members exist, which may be left out (the type admits +`UNSET`), how each one is type-checked, and how each is written back +are all read off the annotations, and the shapes are the ones JSON +takes: `int`, `float` (any JSON number), `bool`, `str`, `JSONValue`, a `Literal` of names, `tuple[T, ...]` or `tuple[T1, T2]`, a TypedDict or dataclass record, `Mapping[str, V]`, a `NewType`, and a nested entity, always as `inner: CodecEntity | Opaque`, because that is @@ -98,7 +108,9 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): registration. A required member has no default; an optional one is `| UNSET = UNSET`, so absence stays distinct from a JSON `null`, and a member that means something when -absent is read that way where it is used, not defaulted. A member is +absent is read that way where it is used, not defaulted. Only `| UNSET` +makes a member optional to a document: a plain default serves hand +construction, and a document must still write the member. A member is read as `codec.configuration.acceleration`; an entity that wants it at the top level adds a `@property` for it. `with_configuration(**changes)` is the entity with members of its configuration replaced, checked as @@ -138,20 +150,30 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): - A codec: its kind is its base class. An `ArrayArrayCodec` defines `transition(incoming: ArrayParts) -> ArrayParts | None` -- abstract: return `incoming` if it leaves the array's shape, grid and data type - alone, or the parts it hands the next codec -- and any codec may define - `incoming_problems(incoming)` for what it cannot take. Every codec - declares `variable_size`, whether its output length depends on its - input, which is what keeps a compressor out of a shard's index. -- A data type: `scalar_storage`, one of `StorageClass` (the `bytes` - codec asks it whether an endianness is needed), and - `fill_value_problems(value, loc)`, abstract: it judges a document's + alone, the parts it hands the next codec, or None when the metadata + cannot say -- and any codec may define `incoming_problems(incoming: + ArrayParts | None) -> tuple[ValidationProblem, ...]` for what it + cannot take, where None is an array the chain lost track of and the + answer to it is nothing. Every codec declares `variable_size`, whether + its output length depends on its input, which is what keeps a + compressor out of a shard's index. +- A data type: `scalar_storage`, one of `StorageClass` -- + `"single_byte"`, `"multi_byte"`, `"variable_length"` -- which the + method `storage_class()` answers (the `bytes` codec asks it whether an + endianness is needed), and `fill_value_problems(value, loc) -> + tuple[ValidationProblem, ...]`, abstract: it judges a document's `fill_value`, and a type that accepts any says so with `return ()`. + These composition hooks return tuples; a record's `problems` yields. The families `IntegerDataType`, `FloatDataType`, `ComplexDataType` and `NumpyTimeDataType` carry both for the types they cover; a family of your own is a plain subclass that is never registered itself, and passes its class variables down. - A chunk grid: `grid(array_shape)`, abstract, and `shape_problems`; see `ChunkGridEntity`. +- A family, one class for many names: `identifier` is an invented key no + document writes, `accepts(name)` says which names are its own, a field + marked `Annotated[str, FROM_NAME]` keeps the name as written, and + `name_problems(name)`, a classmethod, holds any rule about it. Registration is the one moment an entity is refused, with a message that says what to write: a class without `@dataclass`, a codec @@ -161,14 +183,18 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): member whose annotation is not a shape JSON takes -- a nested entity without `Opaque` among them -- a `__post_init__` of the entity's own, a class variable a base -annotates and nothing sets, and what a kind leaves abstract. Everything -else an author could get wrong, pyright says in the editor: the fields, -the class variables and the kind's abstract methods are ordinary typed -Python. A scope reads what a class is off the class: its kind is its -base, its key is its `identifier`, so `extended_with` takes the classes -and nothing can be misfiled. +annotates and nothing sets, and what a kind leaves abstract. What is +left, pyright says in the editor: a member of the wrong type, a +`canonical` returning something else, a hook with the wrong signature. +A scope reads what a class is off the class: its kind is its base, its +key is its `identifier`, so `extended_with` takes the classes and +nothing can be misfiled -- and a class whose `identifier` the scope +already has takes the name over, so registering your own `"gzip"` +replaces the package's reading of it. `Context.of(*classes)` is a scope +of exactly those. -Two complete extensions written against this module alone, as tests: +Two complete extensions written against this module alone, as tests in +the repository: `tests/v3/test_acme_affine.py` (an `array_array` codec with a number, an optional member and a nested data type) and `tests/v3/test_acme_decimal.py` (a configured data type with a diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 4145ce8280..e4b4530796 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -794,3 +794,10 @@ def test_error_a_nested_extension_point_may_not_be_declared_ignorable(case: str) document, loc = IGNORABLE_NESTED[case] problems = validate_array_metadata_v3(cast("Any", document)) assert loc in {problem.loc for problem in problems} + + +def test_error_a_pipeline_that_is_not_an_array_is_not_judged_as_empty() -> None: + # Nothing was read, so there is no chain to find an array->bytes + # codec missing from: the one problem is the shape of the field. + problems = validate_array_metadata_v3(cast("Any", {**BASE, "codecs": "bytes"})) + assert [(problem.loc, problem.kind) for problem in problems] == [(("codecs",), "invalid_type")] diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 9bed6face4..83abb628e6 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -267,6 +267,10 @@ def test_all_is_grouped_and_unique() -> None: "FillValue", "Configuration", "Component", + # The configuration record an entity is built from, `BloscOptions`: + # the dataclass whose fields are the members and whose `problems` + # holds the rules. `Configuration` is taken by the TypedDict. + "Options", # The bare role is the coerced entity itself: the dataclass that owns # the extension's type checks, value checks and canonical spelling. # Listed last so a longer role still wins the alternation. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index c8e3d2619e..1fdffe5979 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -15,6 +15,7 @@ from typing import ( Any, ClassVar, + Literal, Self, cast, get_type_hints, @@ -29,9 +30,18 @@ from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._document import read_array_v3 -from zarr_metadata.v3._entity import is_from_name +from zarr_metadata.v3._entity import BytesBytesCodec, Opaque, is_from_name from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS -from zarr_metadata.v3._typed_json import field_hints, is_not_required, is_optional +from zarr_metadata.v3._typed_json import ( + describe, + field_hints, + is_class_var, + is_not_required, + is_optional, + no_writer_leaf, + shape_of, + writer_for, +) from zarr_metadata.v3.chunk_grid.rectilinear import ( RectilinearChunkGrid, ) @@ -826,3 +836,69 @@ def test_the_fields_are_the_public_configuration_type() -> None: assert { key for key, annotation in keys.items() if not is_not_required(annotation) } == required, cls + + +def test_error_an_entity_of_another_kind_is_invalid_not_out_of_scope() -> None: + # `transpose` is in scope, so it is not for another reader to + # resolve; it is an array->array codec where a bytes->bytes one goes. + value = {"name": "transpose", "configuration": {"order": (0,)}} + entity, problems = CORE_AND_EXTENSIONS.coerce(BytesBytesCodec, value, ("codecs", 2)) + assert entity == Opaque(value, "invalid") + assert [(problem.loc, problem.kind, problem.message) for problem in problems] == [ + ( + ("codecs", 2), + "invalid_value", + "expected a BytesBytesCodec, got 'transpose', an ArrayArrayCodec", + ) + ] + + +@pytest.mark.parametrize( + ("annotation", "shape"), + [ + (Literal[True], "bool"), + (Literal[1, 2], "int"), + (Literal["a", "b"], "str"), + (Literal[0, "auto"], None), + ], +) +def test_a_literal_has_the_shape_of_its_values(annotation: object, shape: str | None) -> None: + # `True` is an integer to Python and a boolean to JSON; a literal + # mixing shapes has none, so a union tries it for any value. + assert shape_of(annotation) == shape + + +def test_a_literal_of_mixed_shapes_is_described() -> None: + assert describe(Literal[0, "auto"]) == "one of ('auto', 0)" + + +@pytest.mark.parametrize( + ("annotation", "expected"), + [ + ("ClassVar[str]", True), + ("ClassVar", True), + ("typing.ClassVar[str]", True), + ("t.ClassVar[str]", True), + ("str", False), + ("ClassVarLike[str]", False), + ("Final[ClassVar[str]]", False), + ], +) +def test_a_class_var_is_read_from_any_spelling(annotation: str, expected: bool) -> None: + # Under PEP 649 the annotation is a string, spelled however the + # module imported the name. + assert is_class_var(annotation) is expected + + +def test_field_hints_is_shared_and_so_read_only() -> None: + hints = field_hints(GzipCodec) + assert field_hints(GzipCodec) is hints + with pytest.raises(TypeError, match="does not support item assignment"): + cast("dict[str, object]", hints)["configuration"] = int + + +def test_error_a_fixed_tuple_of_the_wrong_length_is_not_written() -> None: + write = writer_for(tuple[int, str], no_writer_leaf) + assert write is not None + with pytest.raises(TypeError, match="is not a JSON value"): + write((1,)) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 9e5df5c2af..e2597eaa20 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -8,7 +8,7 @@ import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self +from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self, get_args import pytest from typing_extensions import TypedDict @@ -34,6 +34,7 @@ Configured, Context, DataTypeEntity, + Extents, IntegerDataType, Loc, MetadataEntity, @@ -891,3 +892,84 @@ class Unmarked(BytesBytesCodec): with pytest.raises(TypeError, match="declares `configuration` without `Configured`"): CORE_AND_EXTENSIONS.extended_with(Unmarked) + + +@dataclass(frozen=True) +class NestedRecordOptions(Configuration): + depth: int + + +@dataclass(frozen=True) +class NestingOptions(Configuration): + inner: NestedRecordOptions + + +def test_error_a_member_may_not_be_a_configuration() -> None: + # `problems` is asked of the entity's configuration and of nothing + # inside it; a `Configuration` nested there would carry rules nothing + # runs. + @dataclass(frozen=True) + class Nesting(BytesBytesCodec, Configured): + configuration: NestingOptions + + identifier: ClassVar[str] = "acme.nesting" + variable_size: ClassVar[bool] = False + + with pytest.raises(TypeError, match="a Configuration, whose rules nothing would ask"): + CORE_AND_EXTENSIONS.extended_with(Nesting) + + +@dataclass(frozen=True) +class CheckedOptions(Configuration): + level: int + + def __post_init__(self) -> None: + return None + + +def test_error_a_record_may_not_define_post_init() -> None: + # It would stop at the first problem where `coerce` reports every one. + @dataclass(frozen=True) + class RecordChecked(BytesBytesCodec, Configured): + configuration: CheckedOptions + + identifier: ClassVar[str] = "acme.record_checked" + variable_size: ClassVar[bool] = False + + with pytest.raises( + TypeError, match="CheckedOptions defines __post_init__; write its rules as `problems`" + ): + CORE_AND_EXTENSIONS.extended_with(RecordChecked) + + +@dataclass(frozen=True) +class Window: + start: int + + def __post_init__(self) -> None: + return None + + +@dataclass(frozen=True) +class WindowedOptions(Configuration): + window: Window + + +def test_error_a_nested_record_may_not_define_post_init() -> None: + # A plain record is data, built whenever its keys read; a rule about + # it belongs with the other rules, in the configuration's `problems`. + @dataclass(frozen=True) + class Windowed(BytesBytesCodec, Configured): + configuration: WindowedOptions + + identifier: ClassVar[str] = "acme.windowed" + variable_size: ClassVar[bool] = False + + with pytest.raises(TypeError, match="Window defines __post_init__; a record is plain data"): + CORE_AND_EXTENSIONS.extended_with(Windowed) + + +def test_extents_is_a_type_an_extension_can_write() -> None: + # `Extents | None` in a third party's annotation needs a real alias, + # not a string one. + assert get_args(Extents | None) == (Extents, type(None)) From 1e53667bb704585fcfe1fe8ca87520e03085054c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 09:09:34 +0200 Subject: [PATCH 095/107] fix(zarr-metadata): the document writer puts the envelope back as it was written `to_json` claimed fidelity and `canonical` claimed to be the one transformation, but `to_json` already canonicalized the envelope: an object around a bare name, an empty `configuration` and a `must_understand` of `true` all came back simplified, while an `Opaque` kept them verbatim. zarr-python writes `{"name": "crc32c"}` into every sharded array, so a read-then-write respelled nearly every file. An entity on its own still writes its own spelling, since it has no document to be faithful to. `ArrayDocumentV3.to_json` now walks the document it read beside what its entities write and puts each envelope back as the document spelled it, at any depth, so a document read and written comes out as it went in; what changed is what changes, and an entity put in by hand writes itself. `canonical()` is the one transformation: it rewrites the stored document into each entity's own spelling, so writing a canonical document is the identity. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.8.md | 10 +- .../src/zarr_metadata/v3/_document.py | 126 ++++++++++++++---- .../src/zarr_metadata/v3/_entity.py | 14 +- .../src/zarr_metadata/v3/entity.py | 4 +- .../zarr-metadata/tests/v3/test_entities.py | 64 +++++++++ 5 files changed, 180 insertions(+), 38 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.8.md b/packages/zarr-metadata/changes/4379.feature.8.md index a7bd2d6d74..6b95730c13 100644 --- a/packages/zarr-metadata/changes/4379.feature.8.md +++ b/packages/zarr-metadata/changes/4379.feature.8.md @@ -2,10 +2,12 @@ Serialization and canonicalization are separate operations. `entity.to_json()` writes back what was read, member for member, so a reader that reads a document and writes it returns the bytes it was given. `entity.canonical()` returns the simplest equivalent, and -`canonicalize_array_metadata_v3` is what asks for it. (The envelope's -spelling is the one thing `to_json` does not preserve, because the entity -does not model it: a bare name, `{"name": x}` and -`{"name": x, "configuration": {}}` all read to the same entity.) +`canonicalize_array_metadata_v3` is what asks for it. An entity on its +own spells the envelope its own way, since it does not model it -- a +bare name, `{"name": x}` and `{"name": x, "configuration": {}}` all read +to the same entity -- and `ArrayDocumentV3.to_json` puts back the +spelling the document used, `must_understand` included, so the document +round-trips whole and only `canonical` changes it. Added `canonicalize_array_metadata_v3`, which answers with `Canonical[T] | Invalid`: a semantically valid document in its simplest diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 78e37f3ecf..db70b4defb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -93,45 +93,52 @@ def problems(self) -> tuple[ValidationProblem, ...]: def canonical(self) -> ArrayDocumentV3: """This document in the simplest form that means the same thing. - Each entity in its own canonical form, and the one rule that is - the document's own: `dimension_names` of nothing but nulls says - what omitting the field says. A *transformation*, asked for by - `canonicalize_array_metadata_v3`; `to_json` does not apply it. + Each entity in its own canonical form and in its own spelling of + the envelope -- the bare name when nothing is configured, no + `must_understand`, which means what absence means -- and the one + rule that is the document's own: `dimension_names` of nothing + but nulls says what omitting the field says. A *transformation*, + asked for by `canonicalize_array_metadata_v3`; `to_json` does + not apply it. What comes back is a document written that way, so + writing it changes nothing further. """ - document = dict(self.document) - names = document.get("dimension_names") - if isinstance(names, tuple) and all( - entry is None for entry in cast("tuple[object, ...]", names) - ): - del document["dimension_names"] - return replace( + simplified = replace( self, - document=document, data_type=self.data_type.canonical(), chunk_grid=self.chunk_grid.canonical(), chunk_key_encoding=self.chunk_key_encoding.canonical(), codecs=tuple(codec.canonical() for codec in self.codecs), storage_transformers=tuple(entry.canonical() for entry in self.storage_transformers), ) + document = {**self.document, **_rendered(simplified)} + names = document.get("dimension_names") + if isinstance(names, tuple) and all( + entry is None for entry in cast("tuple[object, ...]", names) + ): + del document["dimension_names"] + return replace(simplified, document=document) def to_json(self) -> dict[str, object]: - """The document as it would be written: every entity in its JSON form. - - Faithful to what was read, member for member; the fields that - are not extension points come back exactly as the document had - them, and a field the document did not have is not invented. - Ask `canonical` first for the simplest equivalent spelling. + """The document as it was written, with each entity's members as the entity has them. + + Faithful: a document read and written comes out as it went in, + an entity's envelope included -- `{"name": "crc32c"}` stays an + object, an empty `configuration` and a `must_understand` of + `true` stay written -- because the document knows the spelling + it read and puts it back around what the entity writes. What + changed is what changes: a member replaced through + `with_configuration` is written as the entity now has it, and an + entity put in by hand is written as it writes itself. A field + the document did not have is not invented, and one it wrote as + something no entity could be read from stands as written. Ask + `canonical` first for the simplest equivalent spelling. """ - rendered: dict[str, object] = { - "data_type": self.data_type.to_json(), - "chunk_grid": self.chunk_grid.to_json(), - "chunk_key_encoding": self.chunk_key_encoding.to_json(), - "codecs": tuple(codec.to_json() for codec in self.codecs), - "storage_transformers": tuple(entry.to_json() for entry in self.storage_transformers), - } return { **self.document, - **{key: value for key, value in rendered.items() if key in self.document}, + **{ + key: _as_written(self.document[key], value) + for key, value in _rendered(self).items() + }, } @classmethod @@ -180,6 +187,73 @@ def parts(self) -> ArrayParts: ) +def _rendered(array: ArrayDocumentV3) -> dict[str, object]: + """Each entity field the document has, as its entities write it; one nothing was read from is left out.""" + rendered: dict[str, object] = {} + for key, entity in ( + ("data_type", array.data_type), + ("chunk_grid", array.chunk_grid), + ("chunk_key_encoding", array.chunk_key_encoding), + ): + if key in array.document: + rendered[key] = entity.to_json() + for key, entities in ( + ("codecs", array.codecs), + ("storage_transformers", array.storage_transformers), + ): + if _listed(array.document, key) is not None: + rendered[key] = tuple(entity.to_json() for entity in entities) + return rendered + + +_ENVELOPE_KEYS = frozenset({"name", "configuration"}) +"""What an entity writes around its members; anything else around a name is the document's.""" + + +def _as_written(original: object, rendered: object) -> object: + """`rendered`, an entity's JSON, in the spelling `original`, the document's JSON at the same place, used. + + The envelope's writer, the counterpart of `named_configuration`. An + entity writes its members and its own spelling of the envelope, + since it has no document to be faithful to; the document has, so + the spellings that mean the same come back as they were written: + the object around a bare name, a `configuration` of nothing, a + `must_understand` of `true`. The two trees are walked together, so + a codec inside a shard is dressed as one in the pipeline is. Where + they disagree -- an entity put in or taken out by hand, a name + changed -- the rendered value stands, members and all. + """ + nothing: dict[str, object] = {} + if isinstance(original, Mapping): + before = cast("Mapping[str, object]", original) + if isinstance(rendered, str): + if before.get("name") == rendered: + return { + key: nothing if key == "configuration" else value + for key, value in before.items() + } + return rendered + if isinstance(rendered, Mapping): + after = cast("Mapping[str, object]", rendered) + written = {key: _as_written(before.get(key), value) for key, value in after.items()} + if ( + after.keys() <= _ENVELOPE_KEYS + and "name" in after + and before.get("name") == after["name"] + ): + for key, value in before.items(): + if key not in written: + written[key] = nothing if key == "configuration" else value + return written + if isinstance(original, (list, tuple)) and isinstance(rendered, (list, tuple)): + before = cast("Sequence[object]", original) + after = cast("Sequence[object]", rendered) + if len(before) != len(after): + return tuple(after) + return tuple(_as_written(entry, value) for entry, value in zip(before, after, strict=True)) + return rendered + + def _listed(document: Mapping[str, object], key: str) -> Sequence[object] | None: """What the document lists at `key`; None if it wrote no array there.""" entries = document.get(key) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 9024a2be00..db3465a637 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -642,13 +642,13 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: object otherwise, a contained entity through its own `to_json`, a JSON-valued member copied so the document is not a handle on the entity. Faithful to every member: read a document, write it - back, and those come out as they went in. Ask `canonical` first - if you want the simplest equivalent spelling. The envelope's - spelling is the one thing not preserved, because the entity does - not model it: a bare name, `{"name": x}` and `{"name": x, - "configuration": {}}` all read to the same entity, and a - `must_understand` the document wrote is not written back, since - absent means the same as `true` and `false` is refused. + back, and those come out as they went in. The envelope is + written the entity's way -- the bare name when nothing is + configured, the object otherwise, no `must_understand`, which + means what absence means -- because an entity alone has no + document to be faithful to; `ArrayDocumentV3.to_json` puts back + the spelling the document used. Ask `canonical` first if you + want the simplest equivalent spelling. An entity whose JSON is not its fields overrides this; none in the package does. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index d2dfe8e9db..72a8102cc9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -134,7 +134,9 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): written once in the base, from the record: the bare name when every member is absent, the object otherwise, a contained entity through its own `to_json`; an entity whose JSON is not its fields overrides it, and -none in the package does. `canonical`, the entity in its simplest equivalent form: +none in the package does. `ArrayDocumentV3.to_json` puts each envelope +back as the document spelled it, so a document read and written comes +out as it went in. `canonical`, the entity in its simplest equivalent form: the entity itself by default, overridden where two spellings of its members mean the same, and in an entity that contains entities to put those in canonical form -- `self.with_configuration(inner=self.inner.canonical())`. diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 1fdffe5979..d0c470cf74 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -505,6 +505,70 @@ def test_the_document_writes_back_only_the_fields_it_read() -> None: assert array.to_json() == {"shape": (4,)} +@pytest.mark.parametrize( + "spelling", + [ + "crc32c", + {"name": "crc32c"}, + {"name": "crc32c", "configuration": {}}, + {"name": "crc32c", "must_understand": True}, + {"name": "crc32c", "configuration": {}, "must_understand": True}, + ], + ids=["bare", "object", "empty-configuration", "must-understand", "both"], +) +def test_the_document_writes_an_envelope_as_it_was_written(spelling: object) -> None: + # An entity writes its own spelling; the document knows the one it + # read and puts it back, around a codec in the pipeline and around + # one inside a shard alike. Only `canonical` simplifies it. + document = { + "codecs": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (2,), + "codecs": ("bytes",), + "index_codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + spelling, + ), + }, + }, + spelling, + ), + } + array, problems = read_array_v3(document, CORE_AND_EXTENSIONS) + assert problems == () + assert isinstance(array.codecs[0], ShardingIndexedCodec) + assert isinstance(array.codecs[1], Crc32cCodec) + assert array.to_json() == document + canonical = array.canonical().to_json()["codecs"] + assert isinstance(canonical, tuple) + assert canonical[1] == "crc32c" + + +def test_the_document_writes_a_changed_member_inside_the_envelope_it_read() -> None: + # What changed is what changes: the member is the entity's, the + # spelling around it the document's. An entity put in by hand has + # no spelling on record and writes itself. + document = { + "codecs": ( + "bytes", + {"name": "gzip", "configuration": {"level": 1}, "must_understand": True}, + ), + } + array, problems = read_array_v3(document, CORE_AND_EXTENSIONS) + assert problems == () + bytes_codec, gzip = array.codecs + assert isinstance(gzip, GzipCodec) + changed = dataclasses.replace(array, codecs=(bytes_codec, gzip.with_configuration(level=5))) + assert changed.to_json()["codecs"] == ( + "bytes", + {"name": "gzip", "configuration": {"level": 5}, "must_understand": True}, + ) + swapped = dataclasses.replace(array, codecs=(bytes_codec, Crc32cCodec())) + assert swapped.to_json()["codecs"] == ("bytes", "crc32c") + + def test_an_unreadable_member_is_not_judged_by_its_default() -> None: # `shuffle` could not be read, so it falls back to `noshuffle`, under # which `typesize` means nothing. The absent `typesize` must not be From 9b4665f5b53a1faae7935e03ab56fdc5a8ebab88 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 09:25:10 +0200 Subject: [PATCH 096/107] refactor(zarr-metadata): every entity is a name and a record; Configured is gone `Configured` marked the entities that take a configuration, and the layer branched on it in five places, with a registration refusal to keep the marker and the annotation saying the same thing. The spec makes an absent configuration and an empty one the same, so the branch was never about the metadata: every entity holds a `Configuration`, and a bare-name entity holds the empty one. `MetadataEntity` declares `configuration: Configuration`. An entity with members narrows it to its own record, its one positional argument; an entity of a bare name defaults it to the empty record with `field(default_factory=Configuration)`, keyword-only where a carried name is the positional argument. The plan reads the record or nothing, `coerce` builds every entity the same way, and `with_configuration` lives on the base and refuses an unknown member the way `replace` does. Pyright still catches a `configuration` narrowed to the wrong thing, now for every entity. The base field has no default because pyright refuses a positional child field over a defaulted base field. The default therefore lives on the bare entities, where a forgotten one is a missing argument that pyright reports at the call. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 9 +- packages/zarr-metadata/changes/4379.misc.2.md | 5 +- .../src/zarr_metadata/v3/_entity.py | 233 ++++++++---------- .../v3/chunk_grid/rectilinear.py | 3 +- .../zarr_metadata/v3/chunk_grid/regular.py | 3 +- .../v3/chunk_key_encoding/default.py | 3 +- .../zarr_metadata/v3/chunk_key_encoding/v2.py | 3 +- .../src/zarr_metadata/v3/codec/blosc.py | 3 +- .../src/zarr_metadata/v3/codec/bytes.py | 3 +- .../src/zarr_metadata/v3/codec/cast_value.py | 3 +- .../src/zarr_metadata/v3/codec/crc32c.py | 5 +- .../src/zarr_metadata/v3/codec/gzip.py | 3 +- .../zarr_metadata/v3/codec/scale_offset.py | 3 +- .../v3/codec/sharding_indexed.py | 3 +- .../src/zarr_metadata/v3/codec/transpose.py | 3 +- .../src/zarr_metadata/v3/codec/zstd.py | 3 +- .../zarr_metadata/v3/data_type/_families.py | 11 +- .../src/zarr_metadata/v3/data_type/bool.py | 5 +- .../src/zarr_metadata/v3/data_type/bytes.py | 5 +- .../src/zarr_metadata/v3/data_type/raw.py | 6 +- .../src/zarr_metadata/v3/data_type/string.py | 5 +- .../src/zarr_metadata/v3/data_type/struct.py | 3 +- .../src/zarr_metadata/v3/entity.py | 25 +- .../zarr-metadata/tests/test_public_api.py | 1 - .../tests/v3/test_acme_affine.py | 3 +- .../tests/v3/test_acme_decimal.py | 3 +- .../zarr-metadata/tests/v3/test_entities.py | 18 +- .../tests/v3/test_extension_api.py | 58 ++--- 28 files changed, 207 insertions(+), 224 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 8abbfc26db..4bb9408356 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -30,10 +30,11 @@ a shard's inner codecs are judged whatever its `index_location` says -- and only the judgments the shard itself would make wait on the fix. An entity has the shape of its metadata: its name is the class, and its -configuration, for an entity that adds `Configured` beside its kind, is -a frozen `Configuration` record named in the one field `configuration` --- `class GzipCodec(BytesBytesCodec, Configured)` with `configuration: -GzipOptions`. The type judgment is read off the record's fields rather +configuration is a frozen `Configuration` record named in the one field +`configuration` -- `class GzipCodec(BytesBytesCodec)` with `configuration: +GzipOptions`; an entity of a bare name defaults the field to the empty +record, since the spec makes an absent configuration and an empty one the +same, so there is one rule for every entity. The type judgment is read off the record's fields rather than written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field annotations: `int`, `float` for any number, `bool`, `str`, a `Literal` diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index 10746b5111..a4a5121618 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -136,9 +136,8 @@ document with a shard, six codecs and a nested pipeline reads in 85 µs where it took 199, and one codec in 6 µs where it took 21. An entity has the shape of its metadata. Its name is the class, and its -configuration, for an entity that adds `Configured` beside its kind, is -a frozen `Configuration` record named in the one field `configuration` --- `class GzipCodec(BytesBytesCodec, Configured)` with `configuration: +configuration is a frozen `Configuration` record named in the one field +`configuration` -- `class GzipCodec(BytesBytesCodec)` with `configuration: GzipOptions`, read as `codec.configuration.level` or through a `@property` the entity adds for a member it wants at the top level. The record's fields are the members and its `problems` the rules on them, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index db3465a637..c6c1429064 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -219,82 +219,74 @@ def canonical(self) -> Self: def unreadable(cls: type[MetadataEntity]) -> str | None: """Why `coerce` could not read an instance of `cls`; None if it can. - An entity's fields are `configuration`, which a `Configured` entity - narrows to its own `Configuration` record, and at most one field the - envelope's name fills. Things that type-check cleanly and then go - wrong somewhere that will not name the class: a field of any other - name; a `configuration` on an entity that is not `Configured`; a - configuration that is not a `Configuration`, or a member of it whose - annotation is not a shape JSON takes, or is itself a `Configuration`, - whose rules nothing would ask; a `__post_init__` of the entity's or - its record's own, since the rules go in `problems`; and a class - variable a base annotates and nothing sets -- `identifier` for every - entity, `bounds` for an integer type -- which the first lookup would - fail. Registration asks, and refuses the class with the answer. + An entity's fields are `configuration`, which an entity narrows to + its own `Configuration` record or defaults to the empty one, and at + most one field the envelope's name fills. Things that type-check + cleanly and then go wrong somewhere that will not name the class: a + field of any other name; a configuration that is not a + `Configuration`, or a member of it whose annotation is not a shape + JSON takes, or is itself a `Configuration`, whose rules nothing would + ask; a `__post_init__` of the entity's or its record's own, since the + rules go in `problems`; and a class variable a base annotates and + nothing sets -- `identifier` for every entity, `bounds` for an + integer type -- which the first lookup would fail. Registration asks, + and refuses the class with the answer. """ try: hints = field_hints(cls) except NameError as unresolved: return _unresolved(cls, unresolved) for name, annotation in hints.items(): - if name == "configuration" and issubclass(cls, Configured): + if name == "configuration" or is_from_name(annotation): continue - if is_from_name(annotation): - continue - if name == "configuration": - return ( - f"{cls.__name__} declares `configuration` without `Configured`; an entity with a " - f"configuration adds it beside its kind: class {cls.__name__}(..., Configured)" - ) return ( f"{cls.__name__} declares a field {name!r}; an entity's fields are `configuration`, " "a frozen dataclass of its members, and a name it carries marked FROM_NAME -- put " f"{name!r} in the configuration record" ) - if issubclass(cls, Configured): - record = hints["configuration"] - if not (isinstance(record, type) and issubclass(record, Configuration)): + record = hints["configuration"] + if not (isinstance(record, type) and issubclass(record, Configuration)): + return ( + f"{cls.__name__}: configuration is annotated {record!r}; annotate it with a frozen " + "dataclass subclassing Configuration, one field per configuration member" + ) + try: + members = field_hints(record) + except NameError as unresolved: + return _unresolved(record, unresolved) + if "__post_init__" in vars(record): + return ( + f"{record.__name__} defines __post_init__; write its rules as `problems`, " + "yielding each: the entity's constructor stops at the first, `coerce` reports " + "every one" + ) + unread: list[str] = [] + for name, annotation in members.items(): + inner = without_unset(strip_annotation(annotation)[0]) + if isinstance(inner, type) and issubclass(inner, Configuration): return ( - f"{cls.__name__}: configuration is annotated {record!r}; annotate it with a frozen " - "dataclass subclassing Configuration, one field per configuration member" + f"{cls.__name__}: {name} is annotated {inner.__name__}, a Configuration, whose " + "rules nothing would ask; a member that is an object is a plain record " + "dataclass or a TypedDict, and its rules belong to the entity's configuration" ) try: - members = field_hints(record) + accepted = parser_for(annotation, _nested_field) is not None + except TypeError as refused: + return f"{cls.__name__}: {name} {refused}" except NameError as unresolved: return _unresolved(record, unresolved) - if "__post_init__" in vars(record): - return ( - f"{record.__name__} defines __post_init__; write its rules as `problems`, " - "yielding each: the entity's constructor stops at the first, `coerce` reports " - "every one" - ) - unread: list[str] = [] - for name, annotation in members.items(): - inner = without_unset(strip_annotation(annotation)[0]) - if isinstance(inner, type) and issubclass(inner, Configuration): - return ( - f"{cls.__name__}: {name} is annotated {inner.__name__}, a Configuration, whose " - "rules nothing would ask; a member that is an object is a plain record " - "dataclass or a TypedDict, and its rules belong to the entity's configuration" - ) - try: - accepted = parser_for(annotation, _nested_field) is not None - except TypeError as refused: - return f"{cls.__name__}: {name} {refused}" - except NameError as unresolved: - return _unresolved(record, unresolved) - if not accepted: - unread.append(name) - if len(unread) != 0: - return ( - f"{cls.__name__}: " - f"{'; '.join(f'{name} is annotated {members[name]!r}' for name in unread)}" - ", which is not a shape JSON takes. A member is int, float, bool, str, JSONValue, " - "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " - "record, Mapping[str, V], a NewType, or an entity kind with Opaque " - "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " - "rule in the record's `problems`" - ) + if not accepted: + unread.append(name) + if len(unread) != 0: + return ( + f"{cls.__name__}: " + f"{'; '.join(f'{name} is annotated {members[name]!r}' for name in unread)}" + ", which is not a shape JSON takes. A member is int, float, bool, str, JSONValue, " + "a Literal of names, tuple[T, ...] or tuple[T1, T2], a TypedDict or dataclass " + "record, Mapping[str, V], a NewType, or an entity kind with Opaque " + "(CodecEntity | Opaque); add | UNSET for an optional member, and put any finer " + "rule in the record's `problems`" + ) if "__post_init__" in vars(cls): return ( f"{cls.__name__} defines __post_init__; write its rules as `problems` on its " @@ -403,10 +395,12 @@ class _Plan: from_name: str | None """The field the envelope's name fills, for a family; None for every other entity.""" - parse: Parser[_Reading] | None - """The configuration record's parser; None for an entity that is not `Configured`.""" - write: Callable[[Configured], dict[str, JSONValue]] | None - """The entity's configuration as the JSON object it writes; None for one that is not `Configured`.""" + read: Callable[ + [object, Loc, _Reading], tuple[Configuration | None, tuple[ValidationProblem, ...]] + ] + """The configuration record, built when every member read, and the problems found.""" + write: Callable[[MetadataEntity], dict[str, JSONValue]] + """The entity's configuration as the JSON object it writes; empty for a bare name.""" requires_configuration: bool """Whether the record has a member the document must write.""" @@ -421,19 +415,26 @@ def _plan(cls: type[MetadataEntity]) -> _Plan: """ hints = field_hints(cls) from_name = next((key for key, annotation in hints.items() if is_from_name(annotation)), None) - if not issubclass(cls, Configured): - return _Plan(from_name, None, None, False) record = hints["configuration"] if not (isinstance(record, type) and issubclass(record, Configuration)): # pragma: no cover msg = f"{cls.__name__}: configuration is annotated {record!r}, not a Configuration" raise TypeError(msg) required = any(not is_optional(annotation) for annotation in field_hints(record).values()) + parse = parser(record, _nested_field) writes: RecordWriter = record_writer(record, _nested_field_writer) - def write(entity: Configured) -> dict[str, JSONValue]: + def read( + value: object, loc: Loc, reading: _Reading + ) -> tuple[Configuration | None, tuple[ValidationProblem, ...]]: + typed, found = parse(value, loc, reading) + # The parser builds the record only from an object whose every + # member read; anything else comes back as it came. + return (typed if isinstance(typed, Configuration) else None), found + + def write(entity: MetadataEntity) -> dict[str, JSONValue]: return writes(entity.configuration) - return _Plan(from_name, parser(record, _nested_field), write, required) + return _Plan(from_name, read, write, required) @dataclass(frozen=True) @@ -442,7 +443,10 @@ class Configuration: A frozen dataclass whose fields are the configuration's members, each a shape JSON takes; the entity names it in its `configuration` - field. `problems` is where everything finer than a type goes -- a + field, and an entity of a bare name defaults it to this one, empty, + since the spec makes an absent configuration and an empty one the + same. + `problems` is where everything finer than a type goes -- a bound, a rule about one member, members read together -- yielding each problem as it is found, located relative to the configuration. A reader stops at the first or collects them all, as it needs: the @@ -455,31 +459,6 @@ def problems(self) -> Iterator[ValidationProblem]: yield from () -@dataclass(frozen=True) -class Configured: - """The half of an entity that has a configuration. - - An entity whose metadata carries a `configuration` object adds this - beside its kind -- `class GzipCodec(BytesBytesCodec, Configured)` -- - and narrows the field to its own record: `configuration: - GzipOptions`. What the layer does with a configuration -- parse it, - ask its rules, write it back, replace members of it -- is done here - or asked of this, and an entity that is not `Configured` has none - of it: its metadata is a bare name. - """ - - configuration: Configuration - - def with_configuration(self, **changes: object) -> Self: - """This entity with these configuration members changed. - - `codec.with_configuration(typesize=UNSET)` is the record replaced - member by member and the entity rebuilt around it, so the - constructor checks the result as it checks any other. - """ - return replace(self, configuration=replace(self.configuration, **changes)) - - @dataclass(frozen=True) class MetadataEntity(ABC): """One named entity, coerced from its metadata. @@ -503,10 +482,21 @@ class MetadataEntity(ABC): `problems` holds what the spec says beyond the members' types -- so `BloscCodec(BloscOptions(clevel=99))` raises on the first, and `coerce` reports every one instead -- and writes `canonical` where - two spellings of its members mean the same. `coerce` and `to_json` + two spellings of its members mean the same. An entity of a bare + name defaults the field to the empty record. `coerce` and `to_json` are written once here, against what the record says. """ + configuration: Configuration + """The record of this entity's members. + + An entity with members narrows it to its own record, `configuration: + GzipOptions`, its one positional argument. An entity of a bare name + defaults it to the empty record -- `configuration: Configuration = + field(default_factory=Configuration)` -- so that `Crc32cCodec()` + builds; `coerce` passes the record either way. + """ + identifier: ClassVar[str] """The name this entity is registered under. @@ -531,7 +521,7 @@ def __post_init__(self) -> None: plan = _plan(type(self)) name = self.identifier if plan.from_name is None else getattr(self, plan.from_name) first = next(type(self).name_problems(name), None) - if first is None and isinstance(self, Configured): + if first is None: first = next(self.configuration.problems(), None) if first is not None: raise MetadataValidationError((first,)) @@ -545,6 +535,16 @@ def accepts(cls, name: str) -> bool: """ return name == cls.identifier + def with_configuration(self, **changes: object) -> Self: + """This entity with these configuration members changed. + + `codec.with_configuration(typesize=UNSET)` is the record replaced + member by member and the entity rebuilt around it, so the + constructor checks the result as it checks any other. A name + that is not a member is refused the way `replace` refuses it. + """ + return replace(self, configuration=replace(self.configuration, **changes)) + @classmethod def coerce(cls, value: object, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one. @@ -564,45 +564,29 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: if len(envelope) != 0: return None, envelope plan = _plan(cls) - members: dict[str, object] = {} - if plan.from_name is not None: - members[plan.from_name] = name - reading = _Reading(context, []) - own: tuple[ValidationProblem, ...] = () - if not issubclass(cls, Configured) or plan.parse is None: - own = tuple( - found - for key in (given or {}) - for found in problem( - ("configuration", key), f"unexpected key {key!r}", "unknown_key" - ) - ) - elif given is None and plan.requires_configuration: + carried: dict[str, str] = {} if plan.from_name is None else {plan.from_name: name} + if given is None and plan.requires_configuration: return None, problem( ("configuration",), f"{cls.identifier!r} requires a configuration", "missing_key", ) - else: - # Arrays as tuples before parsing, so a member holds the - # tuples its type declares, never the lists raw JSON - # arrives as. - members["configuration"], own = plan.parse( - as_tuples({} if given is None else given), ("configuration",), reading - ) + reading = _Reading(context, []) + # Arrays as tuples before parsing, so a member holds the tuples + # its type declares, never the lists raw JSON arrives as. A bare + # name's record has no members, so any key is an unknown one. + record, own = plan.read( + as_tuples({} if given is None else given), ("configuration",), reading + ) found = (*own, *reading.nested) - if any(entry.kind != "unknown_key" for entry in own): + if record is None: # An unknown key is survivable; a member that could not be # read is a hole, and judging around it would be guessing. return None, found # The rules, asked of the name and of the record before anything # is built: a name problem lands on the entity, a configuration # problem under the configuration. - record = members.get("configuration") - refused = ( - *cls.name_problems(name), - *(within((), tuple(record.problems())) if isinstance(record, Configuration) else ()), - ) + refused = (*cls.name_problems(name), *within((), tuple(record.problems()))) if len(refused) != 0: # Values the spec disallows: reported rather than raised, # every one. @@ -614,7 +598,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # entity that would be asked composition questions it cannot # answer. return None, found - return cls(**members), found + return cls(configuration=record, **carried), found def canonical(self) -> Self: """This entity in the simplest form that means the same thing. @@ -658,8 +642,6 @@ def to_json(self) -> ZarrV3MetadataFieldJSON: if plan.from_name is not None: carried = getattr(self, plan.from_name) name = carried if isinstance(carried, str) else name - if not isinstance(self, Configured) or plan.write is None: - return name configuration = plan.write(self) if len(configuration) == 0: return name @@ -806,7 +788,6 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "CodecEntity", "Coerced", "Configuration", - "Configured", "DataTypeEntity", "Loc", "MetadataEntity", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index ad5b8fad60..6d171f27fc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( ChunkGridEntity, Configuration, - Configured, Loc, is_integer, problem, @@ -194,7 +193,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class RectilinearChunkGrid(ChunkGridEntity, Configured): +class RectilinearChunkGrid(ChunkGridEntity): """The `rectilinear` chunk grid, coerced from its metadata.""" configuration: RectilinearChunkGridOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 55261e98b1..0048ff7bec 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( ChunkGridEntity, Configuration, - Configured, problem, ) from zarr_metadata.v3._parts import ChunkGrid @@ -80,7 +79,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class RegularChunkGrid(ChunkGridEntity, Configured): +class RegularChunkGrid(ChunkGridEntity): """The `regular` chunk grid, coerced from its metadata.""" configuration: RegularChunkGridOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index e6a63d051b..674ab89560 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -16,7 +16,6 @@ from zarr_metadata.v3._entity import ( ChunkKeyEncodingEntity, Configuration, - Configured, ) DEFAULT_CHUNK_KEY_ENCODING_NAME: Final = "default" @@ -81,7 +80,7 @@ class DefaultChunkKeyEncodingOptions(Configuration): @dataclass(frozen=True) -class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity, Configured): +class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity): """The `default` chunk key encoding, coerced from its metadata.""" configuration: DefaultChunkKeyEncodingOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index c8512cbf8a..dae1007b1b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -22,7 +22,6 @@ from zarr_metadata.v3._entity import ( ChunkKeyEncodingEntity, Configuration, - Configured, ) V2_CHUNK_KEY_ENCODING_NAME: Final = "v2" @@ -87,7 +86,7 @@ class V2ChunkKeyEncodingOptions(Configuration): @dataclass(frozen=True) -class V2ChunkKeyEncoding(ChunkKeyEncodingEntity, Configured): +class V2ChunkKeyEncoding(ChunkKeyEncodingEntity): """The `v2` chunk key encoding, coerced from its metadata.""" configuration: V2ChunkKeyEncodingOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 522b675728..8d69ac77f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -14,7 +14,6 @@ from zarr_metadata.v3._entity import ( BytesBytesCodec, Configuration, - Configured, ) if TYPE_CHECKING: @@ -133,7 +132,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class BloscCodec(BytesBytesCodec, Configured): +class BloscCodec(BytesBytesCodec): """The `blosc` codec, coerced from its metadata. Everything blosc knows about itself: the shape its metadata takes, the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index fe0d2f0f13..790d4b3c5c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -14,7 +14,6 @@ from zarr_metadata.v3._entity import ( ArrayBytesCodec, Configuration, - Configured, DataTypeEntity, problem, ) @@ -89,7 +88,7 @@ class BytesOptions(Configuration): @dataclass(frozen=True) -class BytesCodec(ArrayBytesCodec, Configured): +class BytesCodec(ArrayBytesCodec): """The `bytes` codec, coerced from its metadata. `endian` is optional and absent means something: a one-byte data type diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index a4235911e5..2b467bb9f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -15,7 +15,6 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, - Configured, DataTypeEntity, Opaque, ) @@ -138,7 +137,7 @@ class CastValueOptions(Configuration): @dataclass(frozen=True) -class CastValueCodec(ArrayArrayCodec, Configured): +class CastValueCodec(ArrayArrayCodec): """The `cast_value` codec, coerced from its metadata. Holds the data type it casts to, so like `sharding_indexed` it is diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 963734141e..99574d897e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -7,13 +7,14 @@ key is absent from the metadata. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ClassVar, Final, Literal, NotRequired from typing_extensions import TypedDict from zarr_metadata.v3._entity import ( BytesBytesCodec, + Configuration, ) CRC32C_CODEC_NAME: Final = "crc32c" @@ -66,5 +67,7 @@ class Crc32cCodec(BytesBytesCodec): The name says everything: a checksum has nothing to configure. """ + configuration: Configuration = field(default_factory=Configuration) + identifier: ClassVar[str] = CRC32C_CODEC_NAME variable_size: ClassVar[bool] = False diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 05190ef6b5..034686bb4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( BytesBytesCodec, Configuration, - Configured, ) if TYPE_CHECKING: @@ -84,7 +83,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class GzipCodec(BytesBytesCodec, Configured): +class GzipCodec(BytesBytesCodec): """The `gzip` codec, coerced from its metadata.""" configuration: GzipOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 0c576f2e6f..65bbcc3293 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -15,7 +15,6 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, - Configured, ) from zarr_metadata.v3._parts import ArrayParts @@ -99,7 +98,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class ScaleOffsetCodec(ArrayArrayCodec, Configured): +class ScaleOffsetCodec(ArrayArrayCodec): """The `scale_offset` codec, coerced from its metadata. Both members are optional and any JSON scalar is well-typed here; what diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index c5f06fe6b1..95cc9d9b63 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -17,7 +17,6 @@ ArrayBytesCodec, CodecEntity, Configuration, - Configured, Opaque, problem, ) @@ -120,7 +119,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class ShardingIndexedCodec(ArrayBytesCodec, Configured): +class ShardingIndexedCodec(ArrayBytesCodec): """The `sharding_indexed` codec, coerced from its metadata. Holds two codec pipelines, so it is one of the few entities that diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 9a5918e5a0..45f02504a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -13,7 +13,6 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, - Configured, problem, ) from zarr_metadata.v3._parts import ArrayParts @@ -88,7 +87,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class TransposeCodec(ArrayArrayCodec, Configured): +class TransposeCodec(ArrayArrayCodec): """The `transpose` codec, coerced from its metadata.""" configuration: TransposeOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index a938de790c..c94b58a41a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -16,7 +16,6 @@ from zarr_metadata.v3._entity import ( BytesBytesCodec, Configuration, - Configured, ) if TYPE_CHECKING: @@ -95,7 +94,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class ZstdCodec(BytesBytesCodec, Configured): +class ZstdCodec(BytesBytesCodec): """The `zstd` codec, coerced from its metadata.""" configuration: ZstdOptions diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index a91b91ee4c..c2734dcde9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -14,13 +14,12 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, ClassVar, Final, Literal, cast from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( Configuration, - Configured, DataTypeEntity, StorageClass, is_integer, @@ -69,6 +68,8 @@ def byte_values(value: object, expected: int | None, loc: Loc) -> tuple[Validati class IntegerDataType(DataTypeEntity): """A fixed-width integer. The width is the whole difference.""" + configuration: Configuration = field(default_factory=Configuration) + bounds: ClassVar[tuple[int, int]] def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: @@ -86,6 +87,8 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP class FloatDataType(DataTypeEntity): """A binary float. A fill value may be a number, a named non-finite, or hex.""" + configuration: Configuration = field(default_factory=Configuration) + scalar_storage: ClassVar[StorageClass] = "multi_byte" hex_parser: ClassVar[Callable[[str], object]] @@ -126,6 +129,8 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP class ComplexDataType(DataTypeEntity): """A complex number: a `[real, imag]` pair of the component float type.""" + configuration: Configuration = field(default_factory=Configuration) + scalar_storage: ClassVar[StorageClass] = "multi_byte" component: ClassVar[type[FloatDataType]] @@ -187,7 +192,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class NumpyTimeDataType(DataTypeEntity, Configured): +class NumpyTimeDataType(DataTypeEntity): """A numpy time scalar: a signed 64-bit count of units, or `NaT`. The two time types share their configuration -- a unit and a scale diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index d240ffdf2d..3bb762b578 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -4,11 +4,12 @@ See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ClassVar, Final, Literal from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + Configuration, DataTypeEntity, Loc, StorageClass, @@ -37,6 +38,8 @@ class BoolDataType(DataTypeEntity): """The `bool` data type. The name says everything.""" + configuration: Configuration = field(default_factory=Configuration) + scalar_storage: ClassVar[StorageClass] = "single_byte" identifier: ClassVar[str] = BOOL_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index c52e5237e6..d89c09e1b3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -5,11 +5,12 @@ """ import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ClassVar, Final, Literal, NewType from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + Configuration, DataTypeEntity, Loc, StorageClass, @@ -63,6 +64,8 @@ def base64_bytes(value: str) -> Base64Bytes: class BytesDataType(DataTypeEntity): """The `bytes` data type. The name says everything.""" + configuration: Configuration = field(default_factory=Configuration) + scalar_storage: ClassVar[StorageClass] = "variable_length" identifier: ClassVar[str] = BYTES_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index c39e24bc3f..86158d3737 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -9,12 +9,13 @@ """ import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Annotated, ClassVar, Final, NewType from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( FROM_NAME, + Configuration, DataTypeEntity, Loc, StorageClass, @@ -98,6 +99,9 @@ class RawBytesDataType(DataTypeEntity): `r8`, and canonicalizing it away is not this package's call. """ + # Keyword-only, so the carried name stays the one positional argument. + configuration: Configuration = field(default_factory=Configuration, kw_only=True) + data_type_name: Annotated[str, FROM_NAME] """The spelling as written -- `r8`, `r008` -- which is where the width lives.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 826273607c..02ca4a7dcd 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -4,11 +4,12 @@ See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ClassVar, Final, Literal from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ( + Configuration, DataTypeEntity, Loc, StorageClass, @@ -37,6 +38,8 @@ class StringDataType(DataTypeEntity): """The `string` data type. The name says everything.""" + configuration: Configuration = field(default_factory=Configuration) + scalar_storage: ClassVar[StorageClass] = "variable_length" identifier: ClassVar[str] = STRING_DATA_TYPE_NAME diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 22dd9d9691..88e03852bf 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -15,7 +15,6 @@ from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( Configuration, - Configured, DataTypeEntity, Loc, Opaque, @@ -143,7 +142,7 @@ def problems(self) -> "Iterator[ValidationProblem]": @dataclass(frozen=True) -class StructDataType(DataTypeEntity, Configured): +class StructDataType(DataTypeEntity): """The `struct` data type, coerced from its metadata. A record of named fields, each with a data type of its own -- so this diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 72a8102cc9..4d6e6a9d29 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -41,11 +41,12 @@ **Writing an extension.** Subclass the kind of thing it is -- a codec's kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), `DataTypeEntity`, `ChunkGridEntity`, `ChunkKeyEncodingEntity` or -`StorageTransformerEntity`, with `Configured` beside it if the metadata -carries a configuration; declare that configuration as a frozen +`StorageTransformerEntity`; declare its configuration as a frozen `Configuration` of its members, with every rule finer than a type in its `problems`, and name it in the entity's one field, `configuration`; -add the class to a scope. Complete, and runnable as written: +add the class to a scope. An entity of a bare name defaults the field +to the empty record: `configuration: Configuration = +field(default_factory=Configuration)`. Complete, and runnable as written: from collections.abc import Iterator from dataclasses import dataclass @@ -57,8 +58,7 @@ UNSET, BytesBytesCodec, Configuration, - Configured, - ValidationProblem, + ValidationProblem, ) @dataclass(frozen=True) # the fields are the schema; frozen, so a configuration is a value @@ -74,7 +74,7 @@ def problems(self) -> Iterator[ValidationProblem]: ) @dataclass(frozen=True) - class AcmeLz4Codec(BytesBytesCodec, Configured): + class AcmeLz4Codec(BytesBytesCodec): configuration: AcmeLz4Options # the shape of the metadata: a name, and a configuration identifier: ClassVar[str] = "acme.lz4" @@ -90,9 +90,10 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): assert validate_array_metadata_v3(document, context=SCOPE) == () An entity has the shape of its metadata: a name, which is the class, -and, for a `Configured` one, a configuration, which is a record -dataclass named in the one field `configuration`; an entity of a bare -name is not `Configured` and has no field. The record's fields are the +and a configuration, which is a record dataclass named in the one field +`configuration`; an entity of a bare name defaults it to the empty +`Configuration`, since the spec makes an absent configuration and an +empty one the same. The record's fields are the one place the entity's members are declared; the public `*Configuration` TypedDict beside it declares the JSON, and a test holds the two to the same keys. Which members exist, which may be left out (the type admits @@ -180,8 +181,8 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): Registration is the one moment an entity is refused, with a message that says what to write: a class without `@dataclass`, a codec subclassing `CodecEntity` instead of a kind, a field other than -`configuration` and a carried name, a `configuration` without -`Configured`, a configuration that is not a `Configuration` record, a +`configuration` and a carried name, a configuration that is not a +`Configuration` record, a member whose annotation is not a shape JSON takes -- a nested entity without `Opaque` among them -- a `__post_init__` of the entity's own, a class variable a base @@ -230,7 +231,6 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): CodecEntity, Coerced, Configuration, - Configured, DataTypeEntity, Loc, MetadataEntity, @@ -268,7 +268,6 @@ class AcmeLz4Codec(BytesBytesCodec, Configured): "Coerced", "ComplexDataType", "Configuration", - "Configured", "Context", "DataTypeEntity", "Extents", diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index 83abb628e6..e45e3e86d3 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -300,7 +300,6 @@ def test_all_is_grouped_and_unique() -> None: "Context", "Coerced", "Configuration", - "Configured", "ChunkGrid", "ArrayParts", "ArrayDocumentV3", diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 3234ee43bb..76be17d5c4 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -27,7 +27,6 @@ ArrayDocumentV3, ArrayParts, Configuration, - Configured, DataTypeEntity, MetadataValidationError, Opaque, @@ -75,7 +74,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeAffineCodec(ArrayArrayCodec, Configured): +class AcmeAffineCodec(ArrayArrayCodec): """`x * scale + offset`, stored as `dtype` if one is named.""" configuration: AcmeAffineOptions diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index e860b3590c..16c781fe1a 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -20,7 +20,6 @@ from zarr_metadata.v3.entity import ( Configuration, - Configured, DataTypeEntity, Loc, MetadataValidationError, @@ -97,7 +96,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeDecimalDataType(DataTypeEntity, Configured): +class AcmeDecimalDataType(DataTypeEntity): """The `acme.decimal` data type, coerced from its metadata.""" configuration: AcmeDecimalOptions diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index d0c470cf74..bf10acc896 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -93,7 +93,6 @@ ChunkKeyEncodingEntity, CodecEntity, Configuration, - Configured, DataTypeEntity, MetadataEntity, StorageTransformerEntity, @@ -818,7 +817,7 @@ class AcmeShardCacheOptions(Configuration): @dataclasses.dataclass(frozen=True) -class AcmeShardCache(StorageTransformerEntity, Configured): +class AcmeShardCache(StorageTransformerEntity): """A third-party storage transformer with a member canonical form drops.""" configuration: AcmeShardCacheOptions @@ -902,6 +901,21 @@ def test_the_fields_are_the_public_configuration_type() -> None: } == required, cls +def test_a_bare_name_entity_holds_the_empty_configuration() -> None: + # One rule for every entity: a name and a record. The spec makes an + # absent configuration and an empty one the same, so a bare name + # holds the empty record, reads either spelling, and writes the name. + assert Crc32cCodec().configuration == Configuration() + for spelling in ("crc32c", {"name": "crc32c"}, {"name": "crc32c", "configuration": {}}): + assert CORE_AND_EXTENSIONS.coerce(CodecEntity, spelling) == (Crc32cCodec(), ()) + assert Crc32cCodec().to_json() == "crc32c" + + +def test_error_a_bare_name_entity_has_no_member_to_change() -> None: + with pytest.raises(TypeError, match="unexpected keyword argument 'level'"): + Crc32cCodec().with_configuration(level=1) + + def test_error_an_entity_of_another_kind_is_invalid_not_out_of_scope() -> None: # `transpose` is in scope, so it is not for another reader to # resolve; it is an array->array codec where a bytes->bytes one goes. diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index e2597eaa20..a20d854b96 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -7,7 +7,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Annotated, ClassVar, Literal, NotRequired, Self, get_args import pytest @@ -31,7 +31,6 @@ ChunkKeyEncodingEntity, CodecEntity, Configuration, - Configured, Context, DataTypeEntity, Extents, @@ -64,7 +63,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeLz4Codec(BytesBytesCodec, Configured): +class AcmeLz4Codec(BytesBytesCodec): """A third-party compressor.""" configuration: AcmeLz4Options @@ -81,6 +80,8 @@ def acceleration(self) -> int | UNSET: class AcmeFloat8DataType(DataTypeEntity): """A third-party one-byte float.""" + configuration: Configuration = field(default_factory=Configuration) + identifier: ClassVar[str] = "acme.float8" scalar_storage: ClassVar[StorageClass] = "single_byte" @@ -197,7 +198,7 @@ def test_an_absent_optional_member_is_read_as_unset_whatever_its_default() -> No # `UNSET` in the record, so no field's default decides what a # document said. @dataclass(frozen=True) - class Defaulted(BytesBytesCodec, Configured): + class Defaulted(BytesBytesCodec): configuration: DefaultedOptions identifier: ClassVar[str] = "acme.defaulted" @@ -299,6 +300,7 @@ class Int24DataType(IntegerDataType): class AcmeFixedDataType(DataTypeEntity): """`acme.fixedN`, a fixed-width type for every N.""" + configuration: Configuration = field(default_factory=Configuration, kw_only=True) data_type_name: Annotated[str, FROM_NAME] identifier: ClassVar[str] = "acme.fixed" @@ -352,7 +354,7 @@ def test_error_a_member_needs_a_check_from_somewhere() -> None: # parser, so the entity owes one. Silently skipping the member would # let anything through where the field promised a type. @dataclass(frozen=True) - class Structured(BytesBytesCodec, Configured): + class Structured(BytesBytesCodec): configuration: StructuredOptions identifier: ClassVar[str] = "acme.structured" @@ -373,7 +375,7 @@ class AcmeWrapperOptions(Configuration): @dataclass(frozen=True) -class AcmeWrapperCodec(BytesBytesCodec, Configured): +class AcmeWrapperCodec(BytesBytesCodec): """A codec that applies another codec after its own step.""" configuration: AcmeWrapperOptions @@ -460,7 +462,7 @@ def test_canonical_is_the_entity_s_own_and_reaches_what_it_contains() -> None: # that `noshuffle` ignores, and the frame of 0 that means "unframed" # is dropped -- with nothing to call `super()` for. @dataclass(frozen=True) - class AcmeFramedCodec(BytesBytesCodec, Configured): + class AcmeFramedCodec(BytesBytesCodec): configuration: AcmeFramedOptions identifier: ClassVar[str] = "acme.framed" @@ -500,7 +502,7 @@ def test_error_a_nested_field_names_a_kind() -> None: # `MetadataEntity` is of no kind, so a field typed as one could not # be resolved through any scope. @dataclass(frozen=True) - class Vague(BytesBytesCodec, Configured): + class Vague(BytesBytesCodec): configuration: VagueOptions identifier: ClassVar[str] = "acme.vague" @@ -551,7 +553,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeBlockCodec(BytesBytesCodec, Configured): +class AcmeBlockCodec(BytesBytesCodec): """A codec whose block size must be a power of two.""" configuration: AcmeBlockOptions @@ -605,7 +607,7 @@ def problems(self) -> Iterator[ValidationProblem]: @dataclass(frozen=True) -class AcmeRangeCodec(BytesBytesCodec, Configured): +class AcmeRangeCodec(BytesBytesCodec): """A codec with two rules, so that one can fail after another.""" configuration: AcmeRangeOptions @@ -669,7 +671,7 @@ class Local(TypedDict, closed=True): depth: int @dataclass(frozen=True) - class Localized(BytesBytesCodec, Configured): + class Localized(BytesBytesCodec): configuration: LocalizedOptions identifier: ClassVar[str] = "acme.localized" @@ -720,7 +722,7 @@ def test_a_slotted_entity_is_accepted() -> None: # `@dataclass(slots=True)` builds the class twice; registration sees # the second, whose members are slot descriptors. @dataclass(frozen=True, slots=True) - class AcmeSlotted(BytesBytesCodec, Configured): + class AcmeSlotted(BytesBytesCodec): configuration: AcmeSlottedOptions identifier: ClassVar[str] = "acme.slotted" @@ -740,6 +742,7 @@ def level(self) -> int: def test_a_bare_class_var_is_a_class_variable() -> None: @dataclass(frozen=True) class AcmeNoted(BytesBytesCodec): + configuration: Configuration = field(default_factory=Configuration) identifier: ClassVar[str] = "acme.noted" variable_size: ClassVar[bool] = False note: ClassVar = "not a member" @@ -757,7 +760,7 @@ def test_a_number_member_is_a_float_field() -> None: # point and refuses a bool, which is what a document's `2` and `true` # deserve. @dataclass(frozen=True) - class AcmeScaled(ArrayArrayCodec, Configured): + class AcmeScaled(ArrayArrayCodec): configuration: AcmeScaledOptions identifier: ClassVar[str] = "acme.scaled" @@ -795,7 +798,7 @@ def test_error_an_entity_must_be_a_dataclass() -> None: # Class creation runs before `@dataclass` and cannot see it missing; # registration can, and says so instead of the first `coerce` failing # with the base class's `__init__`. - class Undecorated(BytesBytesCodec, Configured): + class Undecorated(BytesBytesCodec): configuration: UndecoratedOptions identifier: ClassVar[str] = "acme.undecorated" @@ -816,7 +819,7 @@ class ClosedOptions(Configuration): def test_error_a_nested_field_admits_opaque() -> None: # What the field holds when the inner name is out of scope. @dataclass(frozen=True) - class Closed(BytesBytesCodec, Configured): + class Closed(BytesBytesCodec): configuration: ClosedOptions identifier: ClassVar[str] = "acme.closed" @@ -875,25 +878,6 @@ def test_error_a_list_of_problem_tuples_is_refused() -> None: MetadataValidationError([problem(("a",), "bad a")]) # pyright: ignore[reportArgumentType] -@dataclass(frozen=True) -class UnmarkedOptions(Configuration): - level: int - - -def test_error_a_configuration_needs_configured_beside_the_kind() -> None: - # The marker is what the layer branches on: an entity that declares - # the field without it has a field of a name the layer does not read. - @dataclass(frozen=True) - class Unmarked(BytesBytesCodec): - configuration: UnmarkedOptions - - identifier: ClassVar[str] = "acme.unmarked" - variable_size: ClassVar[bool] = False - - with pytest.raises(TypeError, match="declares `configuration` without `Configured`"): - CORE_AND_EXTENSIONS.extended_with(Unmarked) - - @dataclass(frozen=True) class NestedRecordOptions(Configuration): depth: int @@ -909,7 +893,7 @@ def test_error_a_member_may_not_be_a_configuration() -> None: # inside it; a `Configuration` nested there would carry rules nothing # runs. @dataclass(frozen=True) - class Nesting(BytesBytesCodec, Configured): + class Nesting(BytesBytesCodec): configuration: NestingOptions identifier: ClassVar[str] = "acme.nesting" @@ -930,7 +914,7 @@ def __post_init__(self) -> None: def test_error_a_record_may_not_define_post_init() -> None: # It would stop at the first problem where `coerce` reports every one. @dataclass(frozen=True) - class RecordChecked(BytesBytesCodec, Configured): + class RecordChecked(BytesBytesCodec): configuration: CheckedOptions identifier: ClassVar[str] = "acme.record_checked" @@ -959,7 +943,7 @@ def test_error_a_nested_record_may_not_define_post_init() -> None: # A plain record is data, built whenever its keys read; a rule about # it belongs with the other rules, in the configuration's `problems`. @dataclass(frozen=True) - class Windowed(BytesBytesCodec, Configured): + class Windowed(BytesBytesCodec): configuration: WindowedOptions identifier: ClassVar[str] = "acme.windowed" From 60c01c31623f22d97592226fc153a472a34bea66 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 09:44:47 +0200 Subject: [PATCH 097/107] refactor(zarr-metadata): a member is read from the record; the property lifts are gone Every configuration member was declared three times: a field on the record, a key on the public TypedDict, and a `@property` on the entity that returned the field. The property carried no information -- pyright cannot derive it, so each was hand-written -- and seventeen of the twenty-eight were read by nothing in the package. The one read path is now `codec.configuration.level`, the shape the metadata has, and an entity's own methods read `self.configuration.x` the same way. The worked extensions in the tests drop their lifts too, so the door shows one pattern. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4379.misc.2.md | 4 +- .../v3/chunk_grid/rectilinear.py | 24 +++-- .../zarr_metadata/v3/chunk_grid/regular.py | 10 +-- .../v3/chunk_key_encoding/default.py | 4 - .../zarr_metadata/v3/chunk_key_encoding/v2.py | 4 - .../src/zarr_metadata/v3/codec/blosc.py | 22 +---- .../src/zarr_metadata/v3/codec/bytes.py | 6 +- .../src/zarr_metadata/v3/codec/cast_value.py | 20 +---- .../src/zarr_metadata/v3/codec/gzip.py | 4 - .../zarr_metadata/v3/codec/scale_offset.py | 8 -- .../v3/codec/sharding_indexed.py | 38 +++----- .../src/zarr_metadata/v3/codec/transpose.py | 10 +-- .../src/zarr_metadata/v3/codec/zstd.py | 8 -- .../zarr_metadata/v3/data_type/_families.py | 8 -- .../src/zarr_metadata/v3/data_type/struct.py | 13 ++- .../src/zarr_metadata/v3/entity.py | 4 +- .../tests/v3/test_acme_affine.py | 28 +++--- .../tests/v3/test_acme_decimal.py | 18 ++-- .../zarr-metadata/tests/v3/test_entities.py | 6 +- .../tests/v3/test_extension_api.py | 89 ++++--------------- 20 files changed, 73 insertions(+), 255 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.misc.2.md b/packages/zarr-metadata/changes/4379.misc.2.md index a4a5121618..0f0e2c7402 100644 --- a/packages/zarr-metadata/changes/4379.misc.2.md +++ b/packages/zarr-metadata/changes/4379.misc.2.md @@ -138,8 +138,8 @@ where it took 199, and one codec in 6 µs where it took 21. An entity has the shape of its metadata. Its name is the class, and its configuration is a frozen `Configuration` record named in the one field `configuration` -- `class GzipCodec(BytesBytesCodec)` with `configuration: -GzipOptions`, read as `codec.configuration.level` or through a -`@property` the entity adds for a member it wants at the top level. The +GzipOptions`, read as `codec.configuration.level`, the shape the +metadata has. The record's fields are the members and its `problems` the rules on them, so the lift of configuration keys to the entity, which nothing in Python expressed, is gone with the per-member loop in `coerce` that did it: a diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 6d171f27fc..c039e7524c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -200,14 +200,6 @@ class RectilinearChunkGrid(ChunkGridEntity): identifier: ClassVar[str] = RECTILINEAR_CHUNK_GRID_NAME - @property - def kind(self) -> Literal["inline"]: - return self.configuration.kind - - @property - def chunk_shapes(self) -> tuple[RectilinearDimSpec, ...]: - return self.configuration.chunk_shapes - def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """One spec per dimension, and explicit specs must cover it. @@ -218,15 +210,17 @@ def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: if not isinstance(array_shape, (list, tuple)): return () extents = tuple(cast("Sequence[object]", array_shape)) - if len(self.chunk_shapes) != len(extents): + if len(self.configuration.chunk_shapes) != len(extents): return problem( ("chunk_shapes",), - f"chunk_shapes has {len(self.chunk_shapes)} entries but shape has " + f"chunk_shapes has {len(self.configuration.chunk_shapes)} entries but shape has " f"{len(extents)} dimensions", "invalid_value", ) found: list[ValidationProblem] = [] - for dim, (spec, extent) in enumerate(zip(self.chunk_shapes, extents, strict=True)): + for dim, (spec, extent) in enumerate( + zip(self.configuration.chunk_shapes, extents, strict=True) + ): if isinstance(spec, int) or not is_integer(extent): continue total = _covered_extent(spec) @@ -247,7 +241,9 @@ def grid(self, array_shape: object) -> ChunkGrid: axis of `[30, 34]` gives `{30, 34}`, and anything asking about divisibility has to hold for both. """ - return ChunkGrid.derived(tuple(_axis_lengths(spec) for spec in self.chunk_shapes)) + return ChunkGrid.derived( + tuple(_axis_lengths(spec) for spec in self.configuration.chunk_shapes) + ) def canonical(self) -> Self: """Run-length encoded, which is the spelling that does not grow. @@ -255,4 +251,6 @@ def canonical(self) -> Self: Two dimension specs listing the same extents describe the same grid, and the encoded one stays the same size as the array grows. """ - return self.with_configuration(chunk_shapes=canonical_chunk_shapes(self.chunk_shapes)) + return self.with_configuration( + chunk_shapes=canonical_chunk_shapes(self.configuration.chunk_shapes) + ) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 0048ff7bec..a34d65f6a6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -86,24 +86,20 @@ class RegularChunkGrid(ChunkGridEntity): identifier: ClassVar[str] = REGULAR_CHUNK_GRID_NAME - @property - def chunk_shape(self) -> tuple[int, ...]: - return self.configuration.chunk_shape - def shape_problems(self, array_shape: object) -> tuple[ValidationProblem, ...]: """A regular grid must chunk every array dimension.""" if not isinstance(array_shape, (list, tuple)): return () extents = tuple(cast("Sequence[object]", array_shape)) - if len(self.chunk_shape) == len(extents): + if len(self.configuration.chunk_shape) == len(extents): return () return problem( ("chunk_shape",), - f"chunk_shape has {len(self.chunk_shape)} entries but shape has " + f"chunk_shape has {len(self.configuration.chunk_shape)} entries but shape has " f"{len(extents)} dimensions", "invalid_value", ) def grid(self, array_shape: object) -> ChunkGrid: """One extent per axis, the same for every chunk on that axis.""" - return ChunkGrid.regular(self.chunk_shape) + return ChunkGrid.regular(self.configuration.chunk_shape) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index 674ab89560..057d9a9083 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -86,7 +86,3 @@ class DefaultChunkKeyEncoding(ChunkKeyEncodingEntity): configuration: DefaultChunkKeyEncodingOptions identifier: ClassVar[str] = DEFAULT_CHUNK_KEY_ENCODING_NAME - - @property - def separator(self) -> DefaultChunkKeyEncodingSeparator | UNSET: - return self.configuration.separator diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index dae1007b1b..783c02726c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -92,7 +92,3 @@ class V2ChunkKeyEncoding(ChunkKeyEncodingEntity): configuration: V2ChunkKeyEncodingOptions identifier: ClassVar[str] = V2_CHUNK_KEY_ENCODING_NAME - - @property - def separator(self) -> V2ChunkKeyEncodingSeparator | UNSET: - return self.configuration.separator diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 8d69ac77f6..5675663f89 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -148,32 +148,12 @@ class BloscCodec(BytesBytesCodec): # Every member is required but `typesize`, which only means something # when shuffling; `BloscOptions.problems` is where that conditional lives. - @property - def cname(self) -> BloscCName: - return self.configuration.cname - - @property - def clevel(self) -> int: - return self.configuration.clevel - - @property - def shuffle(self) -> BloscShuffle: - return self.configuration.shuffle - - @property - def blocksize(self) -> int: - return self.configuration.blocksize - - @property - def typesize(self) -> int | UNSET: - return self.configuration.typesize - def canonical(self) -> Self: """Without a `typesize` that `noshuffle` renders meaningless. The spec says of that case that "the value is ignored", so two documents differing only there describe the same codec. """ - if self.shuffle != BLOSC_NO_SHUFFLE or self.typesize is UNSET: + if self.configuration.shuffle != BLOSC_NO_SHUFFLE or self.configuration.typesize is UNSET: return self return self.with_configuration(typesize=UNSET) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 790d4b3c5c..b917e3eb9f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -100,10 +100,6 @@ class BytesCodec(ArrayBytesCodec): identifier: ClassVar[str] = BYTES_CODEC_NAME variable_size: ClassVar[bool] = False - @property - def endian(self) -> Endianness | UNSET: - return self.configuration.endian - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """The data type reaching here must have a raw byte representation. @@ -124,7 +120,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb f"bytes codec is not compatible with variable-length data_type {name!r}", "invalid_value", ) - if storage == "multi_byte" and self.endian is UNSET: + if storage == "multi_byte" and self.configuration.endian is UNSET: return problem( ("endian",), f"endian is required for data type {name!r}, which contains multi-byte values", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 2b467bb9f5..3ac7b4de15 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -149,27 +149,11 @@ class CastValueCodec(ArrayArrayCodec): identifier: ClassVar[str] = CAST_VALUE_CODEC_NAME variable_size: ClassVar[bool] = False - @property - def data_type(self) -> DataTypeEntity | Opaque: - return self.configuration.data_type - - @property - def rounding(self) -> CastRoundingMode | UNSET: - return self.configuration.rounding - - @property - def out_of_range(self) -> CastOutOfRangeMode | UNSET: - return self.configuration.out_of_range - - @property - def scalar_map(self) -> ScalarMap | UNSET: - return self.configuration.scalar_map - def canonical(self) -> Self: """The target data type in its own canonical form.""" - return self.with_configuration(data_type=self.data_type.canonical()) + return self.with_configuration(data_type=self.configuration.data_type.canonical()) def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same parts, holding the type this codec casts to.""" - data_type = self.data_type + data_type = self.configuration.data_type return incoming.with_data_type(data_type if isinstance(data_type, DataTypeEntity) else None) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 034686bb4c..9745a6697b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -90,7 +90,3 @@ class GzipCodec(BytesBytesCodec): identifier: ClassVar[str] = GZIP_CODEC_NAME variable_size: ClassVar[bool] = True - - @property - def level(self) -> int: - return self.configuration.level diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 65bbcc3293..6e62c4e31c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -111,14 +111,6 @@ class ScaleOffsetCodec(ArrayArrayCodec): identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME variable_size: ClassVar[bool] = False - @property - def offset(self) -> JSONValue | UNSET: - return self.configuration.offset - - @property - def scale(self) -> JSONValue | UNSET: - return self.configuration.scale - def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same array, element for element. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 95cc9d9b63..9ddc2f1a49 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -132,27 +132,11 @@ class ShardingIndexedCodec(ArrayBytesCodec): identifier: ClassVar[str] = SHARDING_INDEXED_CODEC_NAME variable_size: ClassVar[bool] = True - @property - def chunk_shape(self) -> tuple[int, ...]: - return self.configuration.chunk_shape - - @property - def codecs(self) -> tuple[CodecEntity | Opaque, ...]: - return self.configuration.codecs - - @property - def index_codecs(self) -> tuple[CodecEntity | Opaque, ...]: - return self.configuration.index_codecs - - @property - def index_location(self) -> ShardingIndexLocation | UNSET: - return self.configuration.index_location - def canonical(self) -> Self: """Each pipeline's codecs in their own canonical form.""" return self.with_configuration( - codecs=tuple(codec.canonical() for codec in self.codecs), - index_codecs=tuple(codec.canonical() for codec in self.index_codecs), + codecs=tuple(codec.canonical() for codec in self.configuration.codecs), + index_codecs=tuple(codec.canonical() for codec in self.configuration.index_codecs), ) def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: @@ -172,9 +156,9 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb outer = incoming.grid if incoming is not None else UNKNOWN_GRID found.extend( chain_problems( - self.codecs, + self.configuration.codecs, ArrayParts( - ChunkGrid.regular(self.chunk_shape), + ChunkGrid.regular(self.configuration.chunk_shape), incoming.data_type if incoming is not None else None, ), ("codecs",), @@ -182,8 +166,10 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb ) found.extend( chain_problems( - self.index_codecs, - ArrayParts(shard_index_grid(outer, self.chunk_shape), Uint64DataType()), + self.configuration.index_codecs, + ArrayParts( + shard_index_grid(outer, self.configuration.chunk_shape), Uint64DataType() + ), ("index_codecs",), ) ) @@ -194,7 +180,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb "index_codecs must be fixed-size", "invalid_value", ) - for index, codec in enumerate(self.index_codecs) + for index, codec in enumerate(self.configuration.index_codecs) if isinstance(codec, CodecEntity) and type(codec).variable_size ) return tuple(found) @@ -203,15 +189,15 @@ def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[Validation """Whether the inner chunk divides every chunk this shard receives.""" if incoming is None or incoming.grid.rank is None: return () - if len(self.chunk_shape) != incoming.grid.rank: + if len(self.configuration.chunk_shape) != incoming.grid.rank: return problem( ("chunk_shape",), - f"chunk_shape has {len(self.chunk_shape)} entries but the incoming array " + f"chunk_shape has {len(self.configuration.chunk_shape)} entries but the incoming array " f"has {incoming.grid.rank} dimensions", "invalid_value", ) found: list[ValidationProblem] = [] - for position, extent in enumerate(self.chunk_shape): + for position, extent in enumerate(self.configuration.chunk_shape): lengths = incoming.grid.axis(position) if lengths is None or extent < 1: continue diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index 45f02504a5..0ee986cafc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -95,10 +95,6 @@ class TransposeCodec(ArrayArrayCodec): identifier: ClassVar[str] = TRANSPOSE_CODEC_NAME variable_size: ClassVar[bool] = False - @property - def order(self) -> tuple[int, ...]: - return self.configuration.order - def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """A transpose permutes the array it receives, so ranks must agree. @@ -107,11 +103,11 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb transpose's output. """ rank = incoming.grid.rank if incoming is not None else None - if rank is None or len(self.order) == rank: + if rank is None or len(self.configuration.order) == rank: return () return problem( ("order",), - f"order has {len(self.order)} entries but the incoming array has {rank} dimensions", + f"order has {len(self.configuration.order)} entries but the incoming array has {rank} dimensions", "invalid_value", ) @@ -122,4 +118,4 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: survive the trip; the grid metadata does not, because it is no longer the grid the document wrote. """ - return incoming.with_grid(incoming.grid.permuted(self.order)) + return incoming.with_grid(incoming.grid.permuted(self.configuration.order)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index c94b58a41a..9fc0c76804 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -101,11 +101,3 @@ class ZstdCodec(BytesBytesCodec): identifier: ClassVar[str] = ZSTD_CODEC_NAME variable_size: ClassVar[bool] = True - - @property - def level(self) -> int: - return self.configuration.level - - @property - def checksum(self) -> bool | UNSET: - return self.configuration.checksum diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index c2734dcde9..2ee9cd355d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -204,14 +204,6 @@ class NumpyTimeDataType(DataTypeEntity): scalar_storage: ClassVar[StorageClass] = "multi_byte" - @property - def unit(self) -> NumpyTimeUnit: - return self.configuration.unit - - @property - def scale_factor(self) -> int: - return self.configuration.scale_factor - def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: if value == "NaT": return () diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 88e03852bf..41bf37ab40 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -155,15 +155,12 @@ class StructDataType(DataTypeEntity): identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" - @property - def fields(self) -> tuple[StructFieldComponent, ...]: - return self.configuration.fields - def canonical(self) -> Self: """Each field's data type in its own canonical form.""" return self.with_configuration( fields=tuple( - replace(field, data_type=field.data_type.canonical()) for field in self.fields + replace(field, data_type=field.data_type.canonical()) + for field in self.configuration.fields ), ) @@ -175,7 +172,7 @@ def storage_class(self) -> StorageClass | None: field's type is out of scope: the answer would be a guess. """ widest: StorageClass = "single_byte" - for field in self.fields: + for field in self.configuration.fields: if not isinstance(field.data_type, DataTypeEntity): return None found = field.data_type.storage_class() @@ -203,7 +200,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP ) fills = cast("Mapping[str, object]", value) found: list[ValidationProblem] = [] - for field in self.fields: + for field in self.configuration.fields: at: Loc = (*loc, field.name) if field.name not in fills: found.extend( @@ -215,7 +212,7 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP if not isinstance(field.data_type, DataTypeEntity): continue found.extend(field.data_type.fill_value_problems(fills[field.name], at)) - declared = {field.name for field in self.fields} + declared = {field.name for field in self.configuration.fields} found.extend( ValidationProblem((*loc, key), f"unknown struct fill field {key!r}", "unknown_key") for key in sorted(fills.keys() - declared) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 4d6e6a9d29..04c7a10d5e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -112,8 +112,8 @@ class AcmeLz4Codec(BytesBytesCodec): absent is read that way where it is used, not defaulted. Only `| UNSET` makes a member optional to a document: a plain default serves hand construction, and a document must still write the member. A member is -read as `codec.configuration.acceleration`; an entity that wants it at -the top level adds a `@property` for it. `with_configuration(**changes)` +read as `codec.configuration.acceleration`, the shape the metadata has; +nothing lifts it to the entity. `with_configuration(**changes)` is the entity with members of its configuration replaced, checked as any construction is. diff --git a/packages/zarr-metadata/tests/v3/test_acme_affine.py b/packages/zarr-metadata/tests/v3/test_acme_affine.py index 76be17d5c4..bd52395e0e 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_affine.py +++ b/packages/zarr-metadata/tests/v3/test_acme_affine.py @@ -82,23 +82,13 @@ class AcmeAffineCodec(ArrayArrayCodec): identifier: ClassVar[str] = "acme.affine" variable_size: ClassVar[bool] = False - @property - def scale(self) -> float: - return self.configuration.scale - - @property - def offset(self) -> float | UNSET: - return self.configuration.offset - - @property - def dtype(self) -> DataTypeEntity | Opaque | UNSET: - return self.configuration.dtype - def canonical(self) -> Self: """An offset of 0 is the identity, and absent says the same; `dtype` in its own form.""" return self.with_configuration( - offset=UNSET if self.offset == 0 else self.offset, - dtype=UNSET if self.dtype is UNSET else self.dtype.canonical(), + offset=UNSET if self.configuration.offset == 0 else self.configuration.offset, + dtype=UNSET + if self.configuration.dtype is UNSET + else self.configuration.dtype.canonical(), ) def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: @@ -112,10 +102,12 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb ) def transition(self, incoming: ArrayParts) -> ArrayParts | None: - if self.dtype is UNSET: + if self.configuration.dtype is UNSET: return incoming return incoming.with_data_type( - self.dtype if isinstance(self.dtype, DataTypeEntity) else None + self.configuration.dtype + if isinstance(self.configuration.dtype, DataTypeEntity) + else None ) @@ -207,8 +199,8 @@ def test_an_out_of_scope_dtype_is_kept_and_not_judged() -> None: assert validate_array_metadata_v3(document, context=SCOPE) == () codec = ArrayDocumentV3.from_json(document, context=SCOPE).codecs[0] assert isinstance(codec, AcmeAffineCodec) - assert isinstance(codec.dtype, Opaque) - assert codec.dtype.reason == "out_of_scope" + assert isinstance(codec.configuration.dtype, Opaque) + assert codec.configuration.dtype.reason == "out_of_scope" assert codec.to_json() == entry diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 16c781fe1a..fba83d8107 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -104,14 +104,6 @@ class AcmeDecimalDataType(DataTypeEntity): identifier: ClassVar[str] = ACME_DECIMAL_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "multi_byte" - @property - def precision(self) -> int: - return self.configuration.precision - - @property - def scale(self) -> int: - return self.configuration.scale - def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """A decimal literal whose digits fit `precision` and `scale`. @@ -130,13 +122,13 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP integer_digits = len(matched.group("integer").lstrip("0")) fraction = matched.group("fraction") fraction_digits = 0 if fraction is None else len(fraction) - allowed_integer_digits = self.precision - self.scale + allowed_integer_digits = self.configuration.precision - self.configuration.scale found: list[ValidationProblem] = [] - if fraction_digits > self.scale: + if fraction_digits > self.configuration.scale: found.extend( problem( loc, - f"{value!r} has {fraction_digits} fractional digits, but scale is {self.scale}", + f"{value!r} has {fraction_digits} fractional digits, but scale is {self.configuration.scale}", "invalid_value", ) ) @@ -144,8 +136,8 @@ def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationP found.extend( problem( loc, - f"{value!r} has {integer_digits} integer digits, but precision {self.precision} " - f"with scale {self.scale} allows {allowed_integer_digits}", + f"{value!r} has {integer_digits} integer digits, but precision {self.configuration.precision} " + f"with scale {self.configuration.scale} allows {allowed_integer_digits}", "invalid_value", ) ) diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index bf10acc896..2c5b7388a6 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -339,7 +339,7 @@ def test_an_unknown_key_is_reported_without_losing_the_member() -> None: ) assert [problem.kind for problem in problems] == ["unknown_key"] assert codec is not None - assert codec.scalar_map == {"encode": (), "enc": ()} + assert codec.configuration.scalar_map == {"encode": (), "enc": ()} # (a defect in a nested metadata field, the location it belongs at) @@ -824,10 +824,6 @@ class AcmeShardCache(StorageTransformerEntity): identifier: ClassVar[str] = "acme.shard_cache" - @property - def verbose(self) -> bool | UNSET: - return self.configuration.verbose - def canonical(self) -> Self: return self.with_configuration(verbose=UNSET) diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index a20d854b96..f045a96980 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -71,10 +71,6 @@ class AcmeLz4Codec(BytesBytesCodec): identifier: ClassVar[str] = "acme.lz4" variable_size: ClassVar[bool] = True - @property - def acceleration(self) -> int | UNSET: - return self.configuration.acceleration - @dataclass(frozen=True) class AcmeFloat8DataType(DataTypeEntity): @@ -205,17 +201,13 @@ class Defaulted(BytesBytesCodec): variable_size: ClassVar[bool] = False - @property - def level(self) -> int | UNSET: - return self.configuration.level - - assert Defaulted(DefaultedOptions()).level == 3 + assert Defaulted(DefaultedOptions()).configuration.level == 3 codec, problems = CORE_AND_EXTENSIONS.extended_with(Defaulted).coerce( CodecEntity, "acme.defaulted" ) assert problems == () assert isinstance(codec, Defaulted) - assert codec.level is UNSET + assert codec.configuration.level is UNSET def test_a_reader_gets_entities_or_an_exception() -> None: @@ -276,7 +268,7 @@ def test_a_reader_can_choose_its_own_scope() -> None: assert isinstance(ArrayDocumentV3.from_json(document).codecs[1], Opaque) in_scope = ArrayDocumentV3.from_json(document, context=SCOPE).codecs[1] assert isinstance(in_scope, AcmeLz4Codec) - assert in_scope.acceleration == 4 + assert in_scope.configuration.acceleration == 4 def test_error_a_family_member_must_declare_what_the_family_left_open() -> None: @@ -360,10 +352,6 @@ class Structured(BytesBytesCodec): identifier: ClassVar[str] = "acme.structured" variable_size: ClassVar[bool] = False - @property - def inner(self) -> object: - return self.configuration.inner - with pytest.raises(TypeError, match="inner is annotated .*, which is not a shape JSON takes"): CORE_AND_EXTENSIONS.extended_with(Structured) @@ -384,12 +372,8 @@ class AcmeWrapperCodec(BytesBytesCodec): variable_size: ClassVar[bool] = False - @property - def inner(self) -> CodecEntity | Opaque: - return self.configuration.inner - def canonical(self) -> Self: - return self.with_configuration(inner=self.inner.canonical()) + return self.with_configuration(inner=self.configuration.inner.canonical()) def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: @@ -405,8 +389,9 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: codec, problems = scope.coerce(CodecEntity, entry) assert problems == () assert isinstance(codec, AcmeWrapperCodec) - assert isinstance(codec.inner, GzipCodec) - assert codec.inner.level == 5 + inner = codec.configuration.inner + assert isinstance(inner, GzipCodec) + assert inner.configuration.level == 5 assert codec.to_json() == entry # An inner codec the scope does not model stays verbatim, as anywhere. @@ -414,7 +399,7 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: codec, problems = scope.coerce(CodecEntity, unknown) assert problems == () assert isinstance(codec, AcmeWrapperCodec) - assert isinstance(codec.inner, Opaque) + assert isinstance(codec.configuration.inner, Opaque) assert codec.to_json() == unknown # A problem inside is located inside. @@ -445,9 +430,9 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: } codec, _ = scope.coerce(CodecEntity, verbose) assert isinstance(codec, AcmeWrapperCodec) - inner = codec.canonical().inner + inner = codec.canonical().configuration.inner assert isinstance(inner, BloscCodec) - assert inner.typesize is UNSET + assert inner.configuration.typesize is UNSET @dataclass(frozen=True) @@ -469,18 +454,10 @@ class AcmeFramedCodec(BytesBytesCodec): variable_size: ClassVar[bool] = False - @property - def inner(self) -> CodecEntity | Opaque: - return self.configuration.inner - - @property - def frame(self) -> int | UNSET: - return self.configuration.frame - def canonical(self) -> Self: return self.with_configuration( - inner=self.inner.canonical(), - frame=UNSET if self.frame == 0 else self.frame, + inner=self.configuration.inner.canonical(), + frame=UNSET if self.configuration.frame == 0 else self.configuration.frame, ) blosc = BloscCodec( @@ -490,7 +467,7 @@ def canonical(self) -> Self: assert framed.canonical() == AcmeFramedCodec( AcmeFramedOptions(inner=blosc.with_configuration(typesize=UNSET)) ) - assert framed.inner is blosc # a transformation, not a mutation + assert framed.configuration.inner is blosc # a transformation, not a mutation @dataclass(frozen=True) @@ -508,10 +485,6 @@ class Vague(BytesBytesCodec): identifier: ClassVar[str] = "acme.vague" variable_size: ClassVar[bool] = False - @property - def inner(self) -> MetadataEntity | Opaque: - return self.configuration.inner - with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): CORE_AND_EXTENSIONS.extended_with(Vague) @@ -562,10 +535,6 @@ class AcmeBlockCodec(BytesBytesCodec): variable_size: ClassVar[bool] = False - @property - def block(self) -> int: - return self.configuration.block - def test_a_rule_about_a_member_is_the_record_s_own() -> None: # The rule runs on the typed record and reports relative to the @@ -616,14 +585,6 @@ class AcmeRangeCodec(BytesBytesCodec): variable_size: ClassVar[bool] = False - @property - def low(self) -> int: - return self.configuration.low - - @property - def high(self) -> int: - return self.configuration.high - def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one() -> None: # One method, three consumers: the entity's constructor takes the @@ -666,8 +627,8 @@ class LocalizedOptions(Configuration): def test_error_a_field_annotation_names_what_is_not_defined() -> None: # Annotations are resolved where the class is, at registration; a - # type defined inside a function is not there. - class Local(TypedDict, closed=True): + # type defined inside a function is not there, however real it is. + class Local(TypedDict, closed=True): # pyright: ignore[reportUnusedClass] depth: int @dataclass(frozen=True) @@ -677,10 +638,6 @@ class Localized(BytesBytesCodec): identifier: ClassVar[str] = "acme.localized" variable_size: ClassVar[bool] = False - @property - def inner(self) -> Local: - return self.configuration.inner - with pytest.raises(TypeError, match="a field annotation names 'Local', which is not defined"): CORE_AND_EXTENSIONS.extended_with(Localized) @@ -729,10 +686,6 @@ class AcmeSlotted(BytesBytesCodec): variable_size: ClassVar[bool] = False - @property - def level(self) -> int: - return self.configuration.level - assert AcmeSlotted(AcmeSlottedOptions(level=1)).to_json() == { "name": "acme.slotted", "configuration": {"level": 1}, @@ -767,10 +720,6 @@ class AcmeScaled(ArrayArrayCodec): variable_size: ClassVar[bool] = False - @property - def scale(self) -> float: - return self.configuration.scale - def transition(self, incoming: ArrayParts) -> ArrayParts | None: return incoming @@ -803,10 +752,6 @@ class Undecorated(BytesBytesCodec): identifier: ClassVar[str] = "acme.undecorated" - @property - def level(self) -> int: - return self.configuration.level - with pytest.raises(TypeError, match="not a dataclass; decorate it with @dataclass"): CORE_AND_EXTENSIONS.extended_with(Undecorated) @@ -825,10 +770,6 @@ class Closed(BytesBytesCodec): identifier: ClassVar[str] = "acme.closed" variable_size: ClassVar[bool] = False - @property - def inner(self) -> CodecEntity: - return self.configuration.inner - with pytest.raises(TypeError, match="inner holds an entity but is not written as its kind"): CORE_AND_EXTENSIONS.extended_with(Closed) From 06faf551ffd24d01d8f56c9ed705f60fab245c24 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 10:02:03 +0200 Subject: [PATCH 098/107] fix(zarr-metadata): a record's constructor refuses a member of the wrong type Until now no construction path checked types at runtime. Pyright was the type checker, and `with_configuration(**changes: object)` is the one path it cannot see through: `codec.with_configuration(level=5.0)` built an entity whose rules passed and whose document a reader would refuse. `replace` on a record was the same hole with a longer name. `Configuration.__post_init__` now checks every member against its annotation with the parsers a document is read by, given a leaf for Python values: a member typed `Kind | Opaque` holds an instance of the kind or an `Opaque`, a member typed as a record holds an instance of it, checked in turn, and an optional member may be `UNSET`. An unknown key in a mapping member is the reader's report, not a type, and passes as `coerce` lets it pass. `with_configuration` needs no code of its own: `replace` rebuilds the record through that constructor and the entity through its own, so every path checks types and values. The parser builds the record through the same constructor, so a document read checks types twice. Measured in one process, an unchecked builder for the parser saved 8% of a document read and 15% of a single codec, and was dropped: not a big enough win to be worth an unchecked way to build a record. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems, 0 crashes. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 5 +- .../src/zarr_metadata/v3/_entity.py | 122 ++++++++++++++++-- .../src/zarr_metadata/v3/_typed_json.py | 19 ++- .../src/zarr_metadata/v3/entity.py | 10 +- .../zarr-metadata/tests/v3/test_entities.py | 67 +++++++++- 5 files changed, 196 insertions(+), 27 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 4bb9408356..a098fbe5e2 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -34,7 +34,10 @@ configuration is a frozen `Configuration` record named in the one field `configuration` -- `class GzipCodec(BytesBytesCodec)` with `configuration: GzipOptions`; an entity of a bare name defaults the field to the empty record, since the spec makes an absent configuration and an empty one the -same, so there is one rule for every entity. The type judgment is read off the record's fields rather +same, so there is one rule for every entity. The record's constructor +makes the same type judgment of a value built by hand, so neither +`replace` nor `with_configuration` can smuggle in a member of the wrong +type. The type judgment is read off the record's fields rather than written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field annotations: `int`, `float` for any number, `bool`, `str`, a `Literal` diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index c6c1429064..e3c922ce26 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -29,7 +29,7 @@ import functools from abc import ABC, abstractmethod from collections.abc import Callable, Mapping -from dataclasses import dataclass, replace +from dataclasses import dataclass, is_dataclass, replace from typing import ( TYPE_CHECKING, ClassVar, @@ -41,6 +41,7 @@ get_args, ) +from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem from zarr_metadata.v3._typed_json import ( Loc, @@ -54,9 +55,11 @@ is_integer, is_optional, is_union, + members_of, parser, parser_for, problem, + record_of, record_writer, strip_annotation, without_unset, @@ -254,7 +257,7 @@ def unreadable(cls: type[MetadataEntity]) -> str | None: members = field_hints(record) except NameError as unresolved: return _unresolved(record, unresolved) - if "__post_init__" in vars(record): + if record.__post_init__ is not Configuration.__post_init__: return ( f"{record.__name__} defines __post_init__; write its rules as `problems`, " "yielding each: the entity's constructor stops at the first, `coerce` reports " @@ -389,6 +392,74 @@ def write(value: object) -> JSONValue: return write +def _held_field(annotation: object) -> Parser[None] | None: + """The check on a value a record holds in a field typed as an entity or as a record. + + The leaf the record's constructor checks with, where `_nested_field` + is the one a document is read with: a field typed `Kind | Opaque` + holds an instance of the kind or an `Opaque`, and a field typed as a + record dataclass holds an instance of it, checked in turn. Every + other shape is one JSON takes, and a Python value of that shape + passes the same parser. + """ + kind = nested_kind(annotation) + if kind is not None: + + def holds_entity(value: object, loc: Loc, state: None) -> Parsed: + if isinstance(value, (kind, Opaque)): + return value, () + return value, problem( + loc, f"expected an entity of {kind.__name__} or an Opaque, got {value!r}" + ) + + return holds_entity + inner = without_unset(strip_annotation(annotation)[0]) + if isinstance(inner, type) and is_dataclass(inner): + + def holds_record(value: object, loc: Loc, state: None) -> Parsed: + if not isinstance(value, inner): + return value, problem(loc, f"expected {inner.__name__}, got {value!r}") + return value, tuple( + ValidationProblem((*loc, *found.loc), found.message, found.kind) + for found in mistyped(value) + ) + + return holds_record + return None + + +@functools.cache +def _checks(record: type) -> dict[str, tuple[bool, Parser[None]]]: + """Each field of a record: whether it may be `UNSET`, and the check on a value held in it.""" + return { + name: (is_optional(annotation), parser(annotation, _held_field)) + for name, annotation in field_hints(record).items() + } + + +def mistyped(record: object) -> tuple[ValidationProblem, ...]: + """Every field of `record` holding a value not of its type, located at the field. + + The runtime half of a record's type. Pyright checks the values a + caller writes; this checks the ones it cannot see -- through + `**changes`, through `replace`, through anything typed `object` -- + with the parsers a document is read by, so a record is well-typed + however it was built. + """ + found: list[ValidationProblem] = [] + for name, (optional, check) in _checks(type(record)).items(): + value = getattr(record, name) + if optional and value is UNSET: + continue + # An unknown key in a mapping member is the reader's report, not + # a type: the record holds what the document said, as `coerce` + # lets it, and a hand-built one may say the same. + found.extend( + entry for entry in check(value, (name,), None)[1] if entry.kind != "unknown_key" + ) + return tuple(found) + + @dataclass(frozen=True, slots=True) class _Plan: """How `coerce` reads and `to_json` writes one class, compiled once from its fields.""" @@ -411,7 +482,10 @@ def _plan(cls: type[MetadataEntity]) -> _Plan: Each parser is a function of its annotation alone, taking the reading it runs in as an argument. `TypeError` for a shape no parser reads, - which registration refuses first. + which registration refuses first. The record is built through its + constructor, which checks the members' types once more: the same + work twice, measured at a twelfth of a document read, and the price + of there being no unchecked way to build one. """ hints = field_hints(cls) from_name = next((key for key, annotation in hints.items() if is_from_name(annotation)), None) @@ -420,7 +494,11 @@ def _plan(cls: type[MetadataEntity]) -> _Plan: msg = f"{cls.__name__}: configuration is annotated {record!r}, not a Configuration" raise TypeError(msg) required = any(not is_optional(annotation) for annotation in field_hints(record).values()) - parse = parser(record, _nested_field) + members = members_of(field_hints(record), _nested_field) + if members is None: # pragma: no cover - registration refused the member first + msg = f"{cls.__name__}: a configuration member is not a shape JSON takes" + raise TypeError(msg) + parse = record_of(record, members) writes: RecordWriter = record_writer(record, _nested_field_writer) def read( @@ -452,8 +530,20 @@ class Configuration: A reader stops at the first or collects them all, as it needs: the entity's constructor stops at the first, `coerce` reports every one, and `BloscOptions(...).problems()` answers without an entity at all. + + The constructor refuses a member of the wrong type, so a record is + well-typed however it was built -- by hand, through `replace`, + through an entity's `with_configuration` -- and the rules can trust + what they read. Values are the rules' business, and the entity's + constructor asks them. """ + def __post_init__(self) -> None: + """Refuse every member of the wrong type, so `GzipOptions(level="high")` raises.""" + found = mistyped(self) + if len(found) != 0: + raise MetadataValidationError(found) + def problems(self) -> Iterator[ValidationProblem]: """Every reason these values are not allowed, yielded as found. Default: none.""" yield from () @@ -463,12 +553,13 @@ def problems(self) -> Iterator[ValidationProblem]: class MetadataEntity(ABC): """One named entity, coerced from its metadata. - Subclasses add their configuration members as fields, which is what - makes them well-typed when read: `coerce` builds one only from - metadata it accepted. Built by hand, the types are the caller's - promise -- the record's `problems` judges values, not types. An optional member is - typed `| UNSET` with a default of `UNSET`, so absence is representable - -- and distinct from a `null` the document wrote -- and a canonical + An entity is well-typed and allowed however it was built. `coerce` + builds one only from metadata it accepted; by hand, the record's + constructor refuses a member of the wrong type and the entity's + refuses a value the rules disallow, and `replace` and + `with_configuration` go through both. An optional member is typed + `| UNSET` with a default of `UNSET`, so absence is representable -- + and distinct from a `null` the document wrote -- and a canonical spelling can leave it out. Frozen, so an entity of hashable members is hashable. One holding a @@ -538,10 +629,13 @@ def accepts(cls, name: str) -> bool: def with_configuration(self, **changes: object) -> Self: """This entity with these configuration members changed. - `codec.with_configuration(typesize=UNSET)` is the record replaced - member by member and the entity rebuilt around it, so the - constructor checks the result as it checks any other. A name - that is not a member is refused the way `replace` refuses it. + `codec.with_configuration(typesize=UNSET)` is the record rebuilt + through its constructor, which refuses a member of the wrong + type, and the entity rebuilt through its own, which refuses a + value the rules disallow -- the same checks as any construction, + since pyright cannot see the members through `**changes`. A + name that is not a member is refused the way `replace` refuses + it. """ return replace(self, configuration=replace(self.configuration, **changes)) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index 78313fe8eb..d504fa2df3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -506,8 +506,9 @@ def record_of(record: Callable[..., object], members: Members[S]) -> Parser[S]: """A member that is itself an object with declared keys, built as a dataclass. Built only from an object whose every key read; otherwise the value - comes back as it came, with the reasons. A record is plain data: it - has no rules of its own, so building it cannot fail. + comes back as it came, with the reasons. A record has no rules of + its own, and every member it is built from has just been checked, + so building it cannot fail. """ def parse(value: object, loc: Loc, state: S) -> Parsed: @@ -547,8 +548,13 @@ def parse(candidate: object, loc: Loc, state: S) -> Parsed: # --- the compiler -------------------------------------------------------- -def _members_of(annotations: Mapping[str, object], leaf: Leaf[S]) -> Members[S] | None: - """A member table for an object's keys; None if any key's type has no parser.""" +def members_of(annotations: Mapping[str, object], leaf: Leaf[S]) -> Members[S] | None: + """A member table for an object's keys; None if any key's type has no parser. + + For a caller with a record class of its own to hand to `record_of`; + `parser_for` builds the same table for a record it meets as an + annotation. + """ members: dict[str, tuple[bool, Parser[S]]] = {} for key, annotation in annotations.items(): member = parser_for(annotation, leaf) @@ -630,7 +636,7 @@ def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: if get_origin(inner) is tuple: return _tuple(inner, leaf) if is_typeddict(inner): - members = _members_of(get_type_hints(inner, include_extras=True), leaf) + members = members_of(get_type_hints(inner, include_extras=True), leaf) return None if members is None else object_of(members) if get_origin(inner) in (Mapping, dict): return _mapping(inner, leaf) @@ -650,7 +656,7 @@ def parser_for(annotation: object, leaf: Leaf[S]) -> Parser[S] | None: "about it belongs in the configuration's `problems`" ) raise TypeError(msg) - members = _members_of(field_hints(inner), leaf) + members = members_of(field_hints(inner), leaf) return None if members is None else record_of(inner, members) return None @@ -874,6 +880,7 @@ def record_writer(record: type, leaf: WriterLeaf) -> RecordWriter: "is_union", "keys_of", "mapping_of", + "members_of", "no_leaf", "no_writer_leaf", "object_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 04c7a10d5e..cbdd5f75c7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -123,7 +123,10 @@ class AcmeLz4Codec(BytesBytesCodec): code. Locations are relative to the configuration, and `kind` is `"invalid_value"` for a value rule. The entity's constructor stops at the first problem it yields, so `AcmeLz4Codec(AcmeLz4Options(acceleration=0))` -raises `MetadataValidationError`; `coerce` runs it to the end and +raises `MetadataValidationError`, and the record's constructor refuses +a member of the wrong type, so `AcmeLz4Options(acceleration="fast")` +raises too, whether written by hand, through `replace` or through +`with_configuration`; `coerce` runs it to the end and reports every problem in the document; a reader with a record asks `options.problems()` directly and stops or collects. It runs only on a configuration whose members all read: a member of the wrong type is @@ -187,8 +190,9 @@ class AcmeLz4Codec(BytesBytesCodec): -- a nested entity without `Opaque` among them -- a `__post_init__` of the entity's own, a class variable a base annotates and nothing sets, and what a kind leaves abstract. What is -left, pyright says in the editor: a member of the wrong type, a -`canonical` returning something else, a hook with the wrong signature. +left, pyright says in the editor and the constructors say at runtime: a +member of the wrong type, a value the rules disallow, a `canonical` +returning something else, a hook with the wrong signature. A scope reads what a class is off the class: its kind is its base, its key is its `identifier`, so `extended_with` takes the classes and nothing can be misfiled -- and a class whose `identifier` the scope diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 2c5b7388a6..58bdbfeca0 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -52,14 +52,15 @@ from zarr_metadata.v3.chunk_key_encoding.v2 import ( V2ChunkKeyEncoding, ) -from zarr_metadata.v3.codec.blosc import BloscCodec +from zarr_metadata.v3.codec.blosc import BloscCodec, BloscOptions from zarr_metadata.v3.codec.bytes import BytesCodec from zarr_metadata.v3.codec.cast_value import CastValueCodec from zarr_metadata.v3.codec.crc32c import Crc32cCodec -from zarr_metadata.v3.codec.gzip import GzipCodec +from zarr_metadata.v3.codec.gzip import GzipCodec, GzipOptions from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodec from zarr_metadata.v3.codec.sharding_indexed import ( ShardingIndexedCodec, + ShardingIndexedOptions, ) from zarr_metadata.v3.codec.transpose import TransposeCodec from zarr_metadata.v3.codec.zstd import ZstdCodec @@ -82,7 +83,7 @@ ) from zarr_metadata.v3.data_type.raw import RawBytesDataType from zarr_metadata.v3.data_type.string import StringDataType -from zarr_metadata.v3.data_type.struct import StructDataType +from zarr_metadata.v3.data_type.struct import StructDataType, StructFieldComponent, StructOptions from zarr_metadata.v3.data_type.uint8 import Uint8DataType from zarr_metadata.v3.data_type.uint16 import Uint16DataType from zarr_metadata.v3.data_type.uint32 import Uint32DataType @@ -976,3 +977,63 @@ def test_error_a_fixed_tuple_of_the_wrong_length_is_not_written() -> None: assert write is not None with pytest.raises(TypeError, match="is not a JSON value"): write((1,)) + + +def test_error_a_record_refuses_a_member_of_the_wrong_type() -> None: + # The runtime half of the record's type: pyright sees a value written + # by hand, the constructor sees the rest. + with pytest.raises(MetadataValidationError) as caught: + GzipOptions(level="high") # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind, p.message) for p in caught.value.problems] == [ + (("level",), "invalid_type", "expected an integer, got 'high'") + ] + + +def test_error_a_record_reports_every_mistyped_member() -> None: + with pytest.raises(MetadataValidationError) as caught: + BloscOptions(cname=1, clevel="x", shuffle="shuffle", blocksize=0, typesize=4) # pyright: ignore[reportArgumentType] + assert [p.loc for p in caught.value.problems] == [("cname",), ("clevel",)] + + +def test_error_a_required_member_may_not_be_unset() -> None: + with pytest.raises(MetadataValidationError) as caught: + GzipOptions(level=UNSET) # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind) for p in caught.value.problems] == [(("level",), "invalid_type")] + + +def test_error_with_configuration_refuses_a_member_of_the_wrong_type() -> None: + # `**changes` is typed `object`, so this is the path pyright cannot + # see; the record's constructor sees it, and `5.0` is not an integer. + codec = GzipCodec(GzipOptions(level=1)) + with pytest.raises(MetadataValidationError) as caught: + codec.with_configuration(level=5.0) + assert [(p.loc, p.kind) for p in caught.value.problems] == [(("level",), "invalid_type")] + + +def test_error_a_member_holding_an_entity_holds_one() -> None: + # A shard's pipelines are entities, not the JSON that names them. + with pytest.raises(MetadataValidationError) as caught: + ShardingIndexedOptions( + chunk_shape=(2,), + codecs=("bytes",), # pyright: ignore[reportArgumentType] + index_codecs=(Crc32cCodec(),), + ) + assert [(p.loc, p.kind, p.message) for p in caught.value.problems] == [ + ( + ("codecs", 0), + "invalid_type", + "expected an entity of CodecEntity or an Opaque, got 'bytes'", + ) + ] + + +def test_error_a_member_holding_a_record_holds_one_that_is_well_typed() -> None: + # A record inside a record is checked in turn, and located inside. + with pytest.raises(MetadataValidationError) as caught: + StructOptions(fields=(("a", Int8DataType()),)) # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind) for p in caught.value.problems] == [(("fields", 0), "invalid_type")] + with pytest.raises(MetadataValidationError) as caught: + StructOptions(fields=(StructFieldComponent(name=1, data_type=Int8DataType()),)) # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind) for p in caught.value.problems] == [ + (("fields", 0, "name"), "invalid_type") + ] From 36fd99a05bde10290f0302e15b76e739a5c950a7 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 10:10:57 +0200 Subject: [PATCH 099/107] fix(zarr-metadata): an entity holds its own record, and every reader-built value checks itself The annotations were meant to be the type judgment, and they were, for a document. Built by hand, nothing at runtime held an entity to the record it declares: `GzipCodec(BloscOptions(...))` built, and wrote blosc's members under gzip's name. The entity's constructor now refuses a record that is not its own and a carried name that is not a string, before any rule reads them. The other values the reader builds get the same treatment, so a hand built one is what it says: an `Opaque` refuses a reason the reader does not give, and `ArrayDocumentV3` refuses a field holding anything but an entity of its kind or an `Opaque`, located at the field. One function answers "is this an entity of that kind, or an Opaque" for records and documents alike. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems, 0 crashes. A document read now costs about a tenth more than before the runtime type checks existed, all of it checks that did not exist. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 7 ++- .../src/zarr_metadata/v3/_document.py | 39 ++++++++++++ .../src/zarr_metadata/v3/_entity.py | 59 +++++++++++++++---- .../src/zarr_metadata/v3/entity.py | 5 +- .../zarr-metadata/tests/v3/test_entities.py | 41 +++++++++++++ 5 files changed, 135 insertions(+), 16 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index a098fbe5e2..3b57235d51 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -35,9 +35,10 @@ configuration is a frozen `Configuration` record named in the one field GzipOptions`; an entity of a bare name defaults the field to the empty record, since the spec makes an absent configuration and an empty one the same, so there is one rule for every entity. The record's constructor -makes the same type judgment of a value built by hand, so neither -`replace` nor `with_configuration` can smuggle in a member of the wrong -type. The type judgment is read off the record's fields rather +makes the same type judgment of a value built by hand, and the entity's +holds the record to the one it declares, so neither `replace` nor +`with_configuration` can smuggle in a member of the wrong type or a +record of another entity. The type judgment is read off the record's fields rather than written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field annotations: `int`, `float` for any number, `bool`, `str`, a `Literal` diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index db70b4defb..e454b3e18f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -37,6 +37,8 @@ MetadataEntity, Opaque, StorageTransformerEntity, + held_problems, + problem, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid @@ -69,6 +71,36 @@ class ArrayDocumentV3: codecs: tuple[CodecEntity | Opaque, ...] storage_transformers: tuple[StorageTransformerEntity | Opaque, ...] + def __post_init__(self) -> None: + """Refuse a field holding anything but an entity of its kind or an `Opaque`. + + `read_array_v3` builds a document that holds what it says by + construction; this is the same guarantee for one built by hand, + located at the field. + """ + found = ( + *_as_object(self.document), + *held_problems(self.data_type, DataTypeEntity, ("data_type",)), + *held_problems(self.chunk_grid, ChunkGridEntity, ("chunk_grid",)), + *held_problems( + self.chunk_key_encoding, ChunkKeyEncodingEntity, ("chunk_key_encoding",) + ), + *( + entry + for index, codec in enumerate(self.codecs) + for entry in held_problems(codec, CodecEntity, ("codecs", index)) + ), + *( + entry + for index, transformer in enumerate(self.storage_transformers) + for entry in held_problems( + transformer, StorageTransformerEntity, ("storage_transformers", index) + ) + ), + ) + if len(found) != 0: + raise MetadataValidationError(found) + def problems(self) -> tuple[ValidationProblem, ...]: """Every semantic problem this document has, once it has been read. @@ -187,6 +219,13 @@ def parts(self) -> ArrayParts: ) +def _as_object(value: object) -> tuple[ValidationProblem, ...]: + """Why `value` is not the document as an object, which is what the reader read it as.""" + if isinstance(value, Mapping): + return () + return problem((), f"expected the document as an object, got {value!r}") + + def _rendered(array: ArrayDocumentV3) -> dict[str, object]: """Each entity field the document has, as its entities write it; one nothing was read from is left out.""" rendered: dict[str, object] = {} diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index e3c922ce26..42a37230f3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -200,11 +200,23 @@ class Opaque: Answers `to_json` and `canonical` as an entity does, so a field typed `CodecEntity | Opaque` is written and simplified without asking which case it holds. + + Built by the scope as it reads. The constructor refuses a `reason` + the reader does not give; `json` is whatever the document wrote, + which nothing checks, since the document may have written anything. """ json: object reason: Literal["out_of_scope", "invalid"] + def __post_init__(self) -> None: + """Refuse a reason that is not one of the reader's two.""" + reason: object = self.reason + if reason not in ("out_of_scope", "invalid"): + raise MetadataValidationError( + problem(("reason",), f"expected 'out_of_scope' or 'invalid', got {reason!r}") + ) + def to_json(self) -> ZarrV3MetadataFieldJSON: """The JSON the document wrote, as it wrote it. @@ -392,6 +404,15 @@ def write(value: object) -> JSONValue: return write +def held_problems( + value: object, kind: type[MetadataEntity], loc: Loc +) -> tuple[ValidationProblem, ...]: + """Why `value` is not what a field typed `kind | Opaque` holds: an entity of the kind, or an `Opaque`.""" + if isinstance(value, (kind, Opaque)): + return () + return problem(loc, f"expected an entity of {kind.__name__} or an Opaque, got {value!r}") + + def _held_field(annotation: object) -> Parser[None] | None: """The check on a value a record holds in a field typed as an entity or as a record. @@ -406,11 +427,7 @@ def _held_field(annotation: object) -> Parser[None] | None: if kind is not None: def holds_entity(value: object, loc: Loc, state: None) -> Parsed: - if isinstance(value, (kind, Opaque)): - return value, () - return value, problem( - loc, f"expected an entity of {kind.__name__} or an Opaque, got {value!r}" - ) + return value, held_problems(value, kind, loc) return holds_entity inner = without_unset(strip_annotation(annotation)[0]) @@ -466,6 +483,8 @@ class _Plan: from_name: str | None """The field the envelope's name fills, for a family; None for every other entity.""" + record: type[Configuration] + """The record the entity declares, which its constructor holds it to.""" read: Callable[ [object, Loc, _Reading], tuple[Configuration | None, tuple[ValidationProblem, ...]] ] @@ -512,7 +531,7 @@ def read( def write(entity: MetadataEntity) -> dict[str, JSONValue]: return writes(entity.configuration) - return _Plan(from_name, read, write, required) + return _Plan(from_name, record, read, write, required) @dataclass(frozen=True) @@ -555,9 +574,9 @@ class MetadataEntity(ABC): An entity is well-typed and allowed however it was built. `coerce` builds one only from metadata it accepted; by hand, the record's - constructor refuses a member of the wrong type and the entity's - refuses a value the rules disallow, and `replace` and - `with_configuration` go through both. An optional member is typed + constructor refuses a member of the wrong type, the entity's refuses + a record that is not its own and then a value the rules disallow, + and `replace` and `with_configuration` go through both. An optional member is typed `| UNSET` with a default of `UNSET`, so absence is representable -- and distinct from a `null` the document wrote -- and a canonical spelling can leave it out. @@ -608,9 +627,26 @@ def name_problems(cls, name: str) -> Iterator[ValidationProblem]: yield from () def __post_init__(self) -> None: - """Refuse the first problem the rules find, so `BloscCodec(BloscOptions(clevel=99))` raises.""" + """Refuse a record that is not this entity's own, then the first problem the rules find. + + The runtime half of the entity's type, as the record's constructor + is of the record's: `GzipCodec(BloscOptions(...))` and a family + member carrying a name that is not a string are refused before + any rule reads them. Then `BloscCodec(BloscOptions(clevel=99))` + raises on the first problem the rules yield. + """ plan = _plan(type(self)) - name = self.identifier if plan.from_name is None else getattr(self, plan.from_name) + if not isinstance(self.configuration, plan.record): + raise MetadataValidationError( + problem( + (), + f"expected a {plan.record.__name__} configuration, got " + f"{type(self.configuration).__name__}", + ) + ) + name: object = self.identifier if plan.from_name is None else getattr(self, plan.from_name) + if not isinstance(name, str): + raise MetadataValidationError(problem((), f"expected a string name, got {name!r}")) first = next(type(self).name_problems(name), None) if first is None: first = next(self.configuration.problems(), None) @@ -888,6 +924,7 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "Opaque", "StorageClass", "StorageTransformerEntity", + "held_problems", "is_from_name", "is_integer", "is_metadata_field", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index cbdd5f75c7..6a315a8604 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -191,8 +191,9 @@ class AcmeLz4Codec(BytesBytesCodec): a `__post_init__` of the entity's own, a class variable a base annotates and nothing sets, and what a kind leaves abstract. What is left, pyright says in the editor and the constructors say at runtime: a -member of the wrong type, a value the rules disallow, a `canonical` -returning something else, a hook with the wrong signature. +member of the wrong type, a record that is not the entity's own, a +value the rules disallow, a `canonical` returning something else, a +hook with the wrong signature. A scope reads what a class is off the class: its kind is its base, its key is its `identifier`, so `extended_with` takes the classes and nothing can be misfiled -- and a class whose `identifier` the scope diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 58bdbfeca0..a7185bfb0e 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -1037,3 +1037,44 @@ def test_error_a_member_holding_a_record_holds_one_that_is_well_typed() -> None: assert [(p.loc, p.kind) for p in caught.value.problems] == [ (("fields", 0, "name"), "invalid_type") ] + + +def test_error_an_entity_refuses_a_record_that_is_not_its_own() -> None: + # The runtime half of the entity's type: the record's members are + # well-typed, and they are blosc's, which gzip would write under its + # own name. + with pytest.raises(MetadataValidationError) as caught: + GzipCodec(BloscOptions(cname="zstd", clevel=5, shuffle="noshuffle", blocksize=0)) # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind, p.message) for p in caught.value.problems] == [ + ((), "invalid_type", "expected a GzipOptions configuration, got BloscOptions") + ] + + +def test_error_a_carried_name_is_a_string() -> None: + with pytest.raises(MetadataValidationError) as caught: + RawBytesDataType(16) # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind) for p in caught.value.problems] == [((), "invalid_type")] + + +def test_error_an_opaque_carries_a_reason_the_reader_gives() -> None: + with pytest.raises(MetadataValidationError) as caught: + Opaque({"name": "acme.x"}, "shrug") # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind) for p in caught.value.problems] == [(("reason",), "invalid_type")] + + +def test_error_a_document_holds_an_entity_of_each_field_s_kind() -> None: + # A document built by hand is held to what `read_array_v3` builds: + # each field an entity of its kind, or an `Opaque`. + with pytest.raises(MetadataValidationError) as caught: + ArrayDocumentV3( + document={}, + data_type=GzipCodec(GzipOptions(level=1)), # pyright: ignore[reportArgumentType] + chunk_grid=Opaque(None, "invalid"), + chunk_key_encoding=Opaque(None, "invalid"), + codecs=(Int8DataType(),), # pyright: ignore[reportArgumentType] + storage_transformers=(), + ) + assert [(p.loc, p.kind) for p in caught.value.problems] == [ + (("data_type",), "invalid_type"), + (("codecs", 0), "invalid_type"), + ] From 722c6671b68c2105a0a98599eeb96f786ab3abae Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 10:17:18 +0200 Subject: [PATCH 100/107] perf(zarr-metadata): the reader builds through create_unchecked, having checked The record's and the entity's constructors check types and rules, which is right for anything built by hand and redundant for the reader: the parser has type-checked every member against the same annotations, and `coerce` has run the rules, before either builds. A document read paid for each check twice. `create_unchecked(**fields)` on `Configuration` and on `MetadataEntity` is the one way around the constructors, named for what it is: a caller that has just made the checks builds through it, and nothing else does. The parser builds records through it and `coerce` builds entities, so a read makes each check once. A document read costs what it did before any runtime check existed. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems, 0 crashes. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.7.md | 5 +- .../src/zarr_metadata/v3/_entity.py | 51 +++++++++++++++---- .../src/zarr_metadata/v3/entity.py | 5 +- .../zarr-metadata/tests/v3/test_entities.py | 14 +++++ 4 files changed, 63 insertions(+), 12 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.7.md b/packages/zarr-metadata/changes/4379.feature.7.md index 3b57235d51..aedf54391e 100644 --- a/packages/zarr-metadata/changes/4379.feature.7.md +++ b/packages/zarr-metadata/changes/4379.feature.7.md @@ -38,7 +38,10 @@ same, so there is one rule for every entity. The record's constructor makes the same type judgment of a value built by hand, and the entity's holds the record to the one it declares, so neither `replace` nor `with_configuration` can smuggle in a member of the wrong type or a -record of another entity. The type judgment is read off the record's fields rather +record of another entity. A document read makes each check once: the +parser type-checks the members and `coerce` runs the rules, then both +build through `create_unchecked`, the one path around the constructors, +named for what it is. The type judgment is read off the record's fields rather than written twice. Which members exist, which may be left out (the type admits `UNSET`), and how each is type-checked all follow from the field annotations: `int`, `float` for any number, `bool`, `str`, a `Literal` diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 42a37230f3..46f8269486 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -501,10 +501,9 @@ def _plan(cls: type[MetadataEntity]) -> _Plan: Each parser is a function of its annotation alone, taking the reading it runs in as an argument. `TypeError` for a shape no parser reads, - which registration refuses first. The record is built through its - constructor, which checks the members' types once more: the same - work twice, measured at a twelfth of a document read, and the price - of there being no unchecked way to build one. + which registration refuses first. The record is built through + `create_unchecked`: its members were type-checked by the parsers the + constructor would use, and a read does each check once. """ hints = field_hints(cls) from_name = next((key for key, annotation in hints.items() if is_from_name(annotation)), None) @@ -517,7 +516,7 @@ def _plan(cls: type[MetadataEntity]) -> _Plan: if members is None: # pragma: no cover - registration refused the member first msg = f"{cls.__name__}: a configuration member is not a shape JSON takes" raise TypeError(msg) - parse = record_of(record, members) + parse = record_of(record.create_unchecked, members) writes: RecordWriter = record_writer(record, _nested_field_writer) def read( @@ -563,6 +562,22 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) + @classmethod + def create_unchecked(cls, **members: object) -> Self: + """This record with these members, built without the constructor's check. + + The one way around the check, for a caller that has just made + it: the parser, which type-checked every member against the same + annotations before building the record. Every field is given -- + the parser gives an absent optional member as `UNSET` -- since + nothing here applies a default. Anything that has not checked + the members goes through the constructor. + """ + record = object.__new__(cls) + for name, value in members.items(): + object.__setattr__(record, name, value) + return record + def problems(self) -> Iterator[ValidationProblem]: """Every reason these values are not allowed, yielded as found. Default: none.""" yield from () @@ -573,10 +588,11 @@ class MetadataEntity(ABC): """One named entity, coerced from its metadata. An entity is well-typed and allowed however it was built. `coerce` - builds one only from metadata it accepted; by hand, the record's - constructor refuses a member of the wrong type, the entity's refuses - a record that is not its own and then a value the rules disallow, - and `replace` and `with_configuration` go through both. An optional member is typed + builds one only from metadata it accepted, through `create_unchecked` + once it has; by hand, the record's constructor refuses a member of + the wrong type, the entity's refuses a record that is not its own + and then a value the rules disallow, and `replace` and + `with_configuration` go through both. An optional member is typed `| UNSET` with a default of `UNSET`, so absence is representable -- and distinct from a `null` the document wrote -- and a canonical spelling can leave it out. @@ -662,6 +678,21 @@ def accepts(cls, name: str) -> bool: """ return name == cls.identifier + @classmethod + def create_unchecked(cls, **fields: object) -> Self: + """This entity with these fields, built without the constructor's checks. + + The one way around them, for a caller that has just made them: + `coerce`, which type-checked the record and ran the rules before + building. Every field is given -- the record, and the carried + name for a family -- since nothing here applies a default. + Anything that has not checked goes through the constructor. + """ + entity = object.__new__(cls) + for name, value in fields.items(): + object.__setattr__(entity, name, value) + return entity + def with_configuration(self, **changes: object) -> Self: """This entity with these configuration members changed. @@ -728,7 +759,7 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: # entity that would be asked composition questions it cannot # answer. return None, found - return cls(configuration=record, **carried), found + return cls.create_unchecked(configuration=record, **carried), found def canonical(self) -> Self: """This entity in the simplest form that means the same thing. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 6a315a8604..4179456bcc 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -126,7 +126,10 @@ class AcmeLz4Codec(BytesBytesCodec): raises `MetadataValidationError`, and the record's constructor refuses a member of the wrong type, so `AcmeLz4Options(acceleration="fast")` raises too, whether written by hand, through `replace` or through -`with_configuration`; `coerce` runs it to the end and +`with_configuration`. `create_unchecked(**fields)`, on a record and on +an entity, is the one way around the constructors, for a reader that +has just made their checks: `coerce` is that reader, and a document +read makes each check once. `coerce` runs it to the end and reports every problem in the document; a reader with a record asks `options.problems()` directly and stops or collects. It runs only on a configuration whose members all read: a member of the wrong type is diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index a7185bfb0e..8fb5c27dfd 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -1078,3 +1078,17 @@ def test_error_a_document_holds_an_entity_of_each_field_s_kind() -> None: (("data_type",), "invalid_type"), (("codecs", 0), "invalid_type"), ] + + +def test_create_unchecked_is_the_one_way_around_the_constructors() -> None: + # For a caller that has just made the checks itself, as `coerce` + # has: the record's members are not type-checked, the entity's + # record and rules are not asked. + record = GzipOptions.create_unchecked(level="high") + assert record.level == "high" + codec = GzipCodec.create_unchecked(configuration=GzipOptions(level=99)) + assert codec.configuration.level == 99 + assert codec == GzipCodec.create_unchecked(configuration=GzipOptions(level=99)) + read, problems = GzipCodec.coerce({"name": "gzip", "configuration": {"level": 1}}, CORE) + assert problems == () + assert read == GzipCodec(GzipOptions(level=1)) From 6deae4854cae145cff51ad663b0baacb0f7ac37d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 10:29:24 +0200 Subject: [PATCH 101/107] refactor(zarr-metadata): resolve reads a field; a scope is a value; the class owns its routine The scope was an object with a reading of its own, `Context.coerce`, beside the class's `coerce`, and the two judged the same field differently. The scope is now a value with one question, `claimant(kind, name)`: which class in scope a name belongs to. `resolve(data, kind, context)` is the reader. It relates the identifier in the field to a concrete class through the context, judges the envelope once for every field, and hands the class the field, since the class owns its validation routine. That routine, `coerce`, says what it is: the configuration parsed and the rules asked, the envelope being the field's and `resolve`'s. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.9.md | 12 +- .../src/zarr_metadata/v3/_document.py | 5 +- .../src/zarr_metadata/v3/_entity.py | 113 +++++++++++++-- .../src/zarr_metadata/v3/_registry.py | 134 +++--------------- .../src/zarr_metadata/v3/entity.py | 23 +-- .../tests/rules/test_chain_properties.py | 2 +- .../tests/rules/test_chunk_grid.py | 2 +- .../tests/v3/test_acme_decimal.py | 7 +- .../zarr-metadata/tests/v3/test_entities.py | 36 +++-- .../tests/v3/test_extension_api.py | 57 ++++---- .../tests/v3/test_fill_values.py | 8 +- .../zarr-metadata/tests/v3/test_resolve.py | 8 +- 12 files changed, 217 insertions(+), 190 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.9.md b/packages/zarr-metadata/changes/4379.feature.9.md index c30ce8422c..5070be2354 100644 --- a/packages/zarr-metadata/changes/4379.feature.9.md +++ b/packages/zarr-metadata/changes/4379.feature.9.md @@ -15,14 +15,16 @@ extension point was typed `MetadataEntity | object` -- which *is* `object`, so the union narrowed to nothing and a consumer could not tell a third-party codec from a malformed one. Each field now names its own kind (`DataTypeEntity | Opaque`, `CodecEntity | Opaque`, ...), an -exhaustive two-case union that narrows. `Context.coerce` is overloaded on -the extension point, so it returns the entity type for that point rather -than the base, and the extension-point constants keep their `Literal` -types so a call written with one of them gets the narrow result. +exhaustive two-case union that narrows. `resolve(data, kind, context)`, +the reader, is generic in the kind, so it returns the entity type for +that point rather than the base. It relates the name in `data` to a +class through the context, and that class owns the validation routine, +its `coerce`; the context itself is a value with one question, +`claimant(kind, name)`. A *family* -- one class covering a parameterized set of names, as the raw-byte types cover every `r` -- is registered like anything else. It takes an invented identifier no document can write and claims its own -names through `accepts`, which `resolve` asks when no key matches. There +names through `accepts`, which `claimant` asks of each class. There is no table of spellings anywhere in the package, so a third party can register a family without changing it. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index e454b3e18f..d59738cb5d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -39,6 +39,7 @@ StorageTransformerEntity, held_problems, problem, + resolve, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid @@ -306,7 +307,7 @@ def _read_one( value = document.get(key) if value is None: return Opaque(None, "invalid"), () - return context.coerce(kind, value, (key,), envelope_judged=True) + return resolve(value, kind, context, (key,), envelope_judged=True) def _read_each( @@ -319,7 +320,7 @@ def _read_each( read: list[_EntityT | Opaque] = [] problems: list[ValidationProblem] = [] for index, entry in enumerate(entries): - entity, found = context.coerce(kind, entry, (key, index), envelope_judged=True) + entity, found = resolve(entry, kind, context, (key, index), envelope_judged=True) read.append(entity) problems.extend(found) return tuple(read), tuple(problems) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 46f8269486..9fc89422de 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -42,7 +42,11 @@ ) from zarr_metadata.model._sentinel import UNSET -from zarr_metadata.model._validation import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import ( + MetadataValidationError, + ValidationProblem, + validate_metadata_field_v3, +) from zarr_metadata.v3._typed_json import ( Loc, Parsed, @@ -383,7 +387,7 @@ def parse(value: object, loc: Loc, reading: _Reading) -> Parsed: problems = is_metadata_field(value, loc) if len(problems) != 0: return value, problems - entity, found = reading.context.coerce(kind, value, loc) + entity, found = resolve(value, kind, reading.context, loc) reading.nested.extend(found) return entity, () @@ -404,6 +408,88 @@ def write(value: object) -> JSONValue: return write +def resolve( + data: object, + kind: type[EntityT], + context: Context, + loc: Loc = (), + *, + envelope_judged: bool = False, +) -> tuple[EntityT | Opaque, tuple[ValidationProblem, ...]]: + """`data`, one metadata field, read as an entity of `kind` in `context`. + + The reader. It relates the identifier in `data` to a concrete class + through `context`, and that class owns the validation routine: its + `coerce` is handed the field. What comes back is the entity, or an + `Opaque` saying why not. A name no class in `context` claims is + `out_of_scope` -- an unmodelled extension, left unjudged, which is + what makes the format open. A name claimed and refused, or of + another kind than this position takes, is `invalid`, for the reasons + reported alongside. `loc` prefixes the problems, so they point at + where in the containing configuration the field sat. + + A metadata field is a metadata field wherever it appears, so the + envelope gets the same structural judgment here that the model layer + gives a top-level one -- an extra member, a `configuration` that is + not an object, a `must_understand` that is not a boolean or is + `false`. `envelope_judged` says the model layer has judged and + reported that already, which it has for the fields of a document, so + it is neither judged nor reported twice. The class is asked whenever + there is one to ask -- a stray member or a malformed + `must_understand` says nothing about the configuration -- and not + when the value names no entity or its configuration is not an + object, which the envelope judgment has said. + """ + problems = ( + () + if envelope_judged + else tuple( + ValidationProblem((*loc, *found.loc), found.message, found.kind) + for found in validate_metadata_field_v3(data, allow_must_understand_false=False) + ) + ) + name, _, malformed = named_configuration(data) + if name is None or len(malformed) != 0: + return Opaque(data, "invalid"), problems + # Asked with the kind's kind, so a class of the wrong kind for this + # position is found, and told apart from a name nothing claims. + registered = kind_of(kind) + entity_type = None if registered is None else context.claimant(registered, name) + if entity_type is None: + return Opaque(data, "out_of_scope"), problems + if not issubclass(entity_type, kind): + # In scope, so not for another reader to resolve: the name is an + # entity of the wrong kind for this position. + return Opaque(data, "invalid"), ( + *problems, + ValidationProblem( + loc, + f"expected {_an(kind.__name__)}, got {name!r}, " + f"{_an(_refinement(entity_type, kind).__name__)}", + "invalid_value", + ), + ) + entity, found = entity_type.coerce(data, context) + problems = ( + *problems, + *(ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found), + ) + if entity is None: + return Opaque(data, "invalid"), problems + return entity, problems + + +def _refinement(entity: type[MetadataEntity], kind: type[MetadataEntity]) -> type[MetadataEntity]: + """The class just below `kind`'s kind that `entity` is: `ArrayArrayCodec` for a transpose codec.""" + mro = entity.__mro__ + return mro[mro.index(kind_of(kind) or MetadataEntity) - 1] + + +def _an(noun: str) -> str: + """`noun` with its indefinite article: `an ArrayArrayCodec`, `a BytesBytesCodec`.""" + return f"an {noun}" if noun[:1].upper() in "AEIOU" else f"a {noun}" + + def held_problems( value: object, kind: type[MetadataEntity], loc: Loc ) -> tuple[ValidationProblem, ...]: @@ -708,16 +794,23 @@ def with_configuration(self, **changes: object) -> Self: @classmethod def coerce(cls, value: object, context: Context) -> Coerced[Self]: - """`value` as this entity, or the reasons it is not one. - - The configuration is parsed against the record the `configuration` - field names, member by member; a member holding another entity is - read in `context`, the scope this reading is happening in. An - optional member the document left out is `UNSET` in the record, - so no field's default decides what a document said. The entity is + """`value` as this entity, or the reasons it is not one: the class's validation routine. + + `resolve` relates a field's name to this class and hands it the + field; this is what the class does with it. The configuration is + parsed against the record the `configuration` field names, + member by member; a member holding another entity is read in + `context`, the scope this reading is happening in. An optional + member the document left out is `UNSET` in the record, so no + field's default decides what a document said. The entity is built only when every member of its own read -- its rules are written over a whole configuration -- and handed back only when everything inside it read too. + + The envelope is the field's, not the class's, and `resolve` + judges it: a stray member or a `must_understand` of `false` is + not reported here. Called on a class no scope has registered, + this runs with none of registration's refusals having happened. """ name, given, envelope = named_configuration(value) if name is None or not cls.accepts(name): @@ -950,6 +1043,7 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "Coerced", "Configuration", "DataTypeEntity", + "EntityT", "Loc", "MetadataEntity", "Opaque", @@ -963,6 +1057,7 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "named_configuration", "nested_kind", "problem", + "resolve", "unreadable", "within", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py index 1118711c4d..6486984489 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_registry.py @@ -27,22 +27,17 @@ from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeVar +from typing import Final -from zarr_metadata.model._validation import ( - ValidationProblem, - validate_metadata_field_v3, -) from zarr_metadata.v3._entity import ( KINDS, ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, CodecEntity, + EntityT, MetadataEntity, - Opaque, kind_of, - named_configuration, unreadable, ) from zarr_metadata.v3._typed_json import is_class_var, own_annotations @@ -80,11 +75,6 @@ from zarr_metadata.v3.data_type.uint32 import Uint32DataType from zarr_metadata.v3.data_type.uint64 import Uint64DataType -if TYPE_CHECKING: - from zarr_metadata.v3._entity import Loc - -_EntityT = TypeVar("_EntityT", bound=MetadataEntity) - Tables = Mapping[type[MetadataEntity], Mapping[str, type[MetadataEntity]]] """By kind, then by the identifier each entity is registered under.""" @@ -93,10 +83,12 @@ class Context: """The entities in scope while metadata is being read. - Built from classes with `Context.of`; extended with more by - `extended_with`. What each class is registered as is read off it -- - its kind is its base class, its key is its `identifier` -- so there - is nothing to misfile. + A value, with no reading of its own: `resolve` reads a field in it, + and `claimant` is the one question it answers, which class a name + belongs to. Built from classes with `Context.of`; extended with more + by `extended_with`. What each class is registered as is read off it + -- its kind is its base class, its key is its `identifier` -- so + there is nothing to misfile. """ tables: Tables @@ -127,109 +119,25 @@ def entities(self) -> tuple[type[MetadataEntity], ...]: """Every entity in scope, kind by kind.""" return tuple(entity for table in self.tables.values() for entity in table.values()) - def resolve(self, kind: type[_EntityT], name: str) -> type[_EntityT] | None: - """The entity of `kind` that `name` denotes, or None if out of scope. + def claimant(self, kind: type[EntityT], name: str) -> type[EntityT] | None: + """The class in scope that claims `name` as an entity of `kind`; None if none does. - `kind` may be a subclass of a kind -- `GzipCodec`, a family -- in - which case only an entity under it resolves. Out of scope is not - an error: an unknown name may be an extension this reader does - not model, and openness means leaving it unjudged. - - Each entity of the kind is asked whether the name is its own, - through `accepts`, in registration order, and the first to claim - it answers for it. A family covers many names with one class, so - a table keyed by name could not hold it; the identifier keys - exist for `extended_with` to take a name over, not for lookup. + Asks each class registered under the kind's kind whether the name + is its own -- a family claims every `r` -- rather than looking + a key up, so the identifier keys exist for `extended_with` to + take a name over, not for lookup. A class that claims the name + but is not a `kind` -- `transpose` asked for as a + `BytesBytesCodec` -- is none; `resolve` asks with the kind's kind + to tell that case from a name nothing claims. """ - entity = self._claimant(kind, name) - if entity is None or not issubclass(entity, kind): - return None - return entity - - def _claimant(self, kind: type[MetadataEntity], name: str) -> type[MetadataEntity] | None: - """The entity registered under `kind`'s kind that claims `name`, whatever its subclass.""" registered = kind_of(kind) if registered is None: return None table = self.tables.get(registered, {}) - return next((candidate for candidate in table.values() if candidate.accepts(name)), None) - - def coerce( - self, - kind: type[_EntityT], - value: object, - loc: Loc = (), - *, - envelope_judged: bool = False, - ) -> tuple[_EntityT | Opaque, tuple[ValidationProblem, ...]]: - """One nested entity of `kind`, read in this scope. - - The primitive the containing entities are built from: a `struct` - data type reads its fields with it, a `sharding_indexed` codec its - two pipelines. Returns the entity when its name is in scope, and - the value untouched when it is not -- an unmodelled extension is - left unjudged, which is what makes the format open. - - `loc` prefixes the problems, so they point at where in the - containing configuration the entity sat. - - A metadata field is a metadata field wherever it appears, so the - envelope gets the same structural judgment here that the model - layer gives a top-level one -- an extra member, a `configuration` - that is not an object, a `must_understand` that is not a boolean - or is `false`. `envelope_judged` says the model layer has judged - and reported that already, which it has for the fields of a - document, so it is neither judged nor reported twice. The entity - is read whenever there is one to read -- a stray member or a - malformed `must_understand` says nothing about the configuration - -- and not when the value names no entity or its configuration - is not an object, which the envelope judgment has said. - """ - problems = ( - () - if envelope_judged - else tuple( - ValidationProblem((*loc, *found.loc), found.message, found.kind) - for found in validate_metadata_field_v3(value, allow_must_understand_false=False) - ) - ) - name, _, malformed = named_configuration(value) - if name is None or len(malformed) != 0: - return Opaque(value, "invalid"), problems - entity_type = self._claimant(kind, name) - if entity_type is None: - return Opaque(value, "out_of_scope"), problems - if not issubclass(entity_type, kind): - # In scope, so not for another reader to resolve: the name - # is an entity of the wrong kind for this position. - return Opaque(value, "invalid"), ( - *problems, - ValidationProblem( - loc, - f"expected {_an(kind.__name__)}, got {name!r}, " - f"{_an(_refinement(entity_type, kind).__name__)}", - "invalid_value", - ), - ) - entity, found = entity_type.coerce(value, self) - problems = ( - *problems, - *(ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found), - ) - if entity is None: - return Opaque(value, "invalid"), problems - return entity, problems - - -def _refinement(entity: type[MetadataEntity], kind: type[MetadataEntity]) -> type[MetadataEntity]: - """The class just below `kind`'s kind that `entity` is: `ArrayArrayCodec` for a transpose codec.""" - mro = entity.__mro__ - return mro[mro.index(kind_of(kind) or MetadataEntity) - 1] - - -def _an(noun: str) -> str: - """`noun` with its indefinite article: `an ArrayArrayCodec`, `a BytesBytesCodec`.""" - return f"an {noun}" if noun[:1].upper() in "AEIOU" else f"a {noun}" + entity = next((candidate for candidate in table.values() if candidate.accepts(name)), None) + if entity is None or not issubclass(entity, kind): + return None + return entity def _registrable(entity: type[MetadataEntity]) -> type[MetadataEntity]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 4179456bcc..766bbd194a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -30,13 +30,17 @@ indexing into the document: `("codecs", 1, "configuration", "level")`, and `kind` one of `invalid_type`, `invalid_value`, `missing_key`, `unknown_key` and -`invalid_json`. `SCOPE.coerce(CodecEntity, entry)` reads one metadata -field as an entity of that kind and returns `(entity, problems)` where -`entity` is the entity or an `Opaque` -- never `None` -- with `loc` -relative to the entry: `("configuration", "level")`. An entity's own -`coerce(value, context)` returns `(entity or None, problems)`; that is -`Coerced`. Constructing an entity by hand raises `MetadataValidationError` -with `loc` relative to the configuration: `("level",)`. +`invalid_json`. `resolve(entry, CodecEntity, SCOPE)` reads one metadata +field as an entity of that kind: it relates the name in the entry to a +class in the scope and hands that class the field, since the class owns +its validation routine. It returns `(entity, problems)` where `entity` +is the entity or an `Opaque` -- never `None` -- with `loc` relative to +the entry: `("configuration", "level")`. The class's routine, +`coerce(value, context)`, returns `(entity or None, problems)`, that is +`Coerced`, and judges the configuration; the envelope is the field's, +and `resolve` judges it. Constructing an entity by hand raises +`MetadataValidationError` with `loc` relative to the configuration: +`("level",)`. **Writing an extension.** Subclass the kind of thing it is -- a codec's kind (`ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec`), @@ -202,7 +206,8 @@ class AcmeLz4Codec(BytesBytesCodec): nothing can be misfiled -- and a class whose `identifier` the scope already has takes the name over, so registering your own `"gzip"` replaces the package's reading of it. `Context.of(*classes)` is a scope -of exactly those. +of exactly those, and a scope is a value: `resolve` reads in it, and +`claimant(kind, name)` says which class a name belongs to. Two complete extensions written against this module alone, as tests in the repository: @@ -248,6 +253,7 @@ class AcmeLz4Codec(BytesBytesCodec): is_integer, named_configuration, problem, + resolve, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid, Extents @@ -296,5 +302,6 @@ class AcmeLz4Codec(BytesBytesCodec): "is_integer", "named_configuration", "problem", + "resolve", "within", ] diff --git a/packages/zarr-metadata/tests/rules/test_chain_properties.py b/packages/zarr-metadata/tests/rules/test_chain_properties.py index 953e55ea90..77da33530b 100644 --- a/packages/zarr-metadata/tests/rules/test_chain_properties.py +++ b/packages/zarr-metadata/tests/rules/test_chain_properties.py @@ -66,7 +66,7 @@ def test_the_strategies_cover_every_codec_the_package_models() -> None: for kinds, expected in ((ARRAY_ARRAY, ArrayArrayCodec), (ARRAY_BYTES, ArrayBytesCodec)): for entry in kinds: name = entry.__annotations__["name"].__args__[0] - entity = CORE_AND_EXTENSIONS.resolve(CodecEntity, name) + entity = CORE_AND_EXTENSIONS.claimant(CodecEntity, name) assert entity is not None, name assert issubclass(entity, expected) diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index ab7d572569..b1d7d67ab8 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -101,7 +101,7 @@ def _grid_of(grid: object, shape: object) -> ChunkGrid: else None ) entity_type = ( - CORE_AND_EXTENSIONS.resolve(ChunkGridEntity, name) if isinstance(name, str) else None + CORE_AND_EXTENSIONS.claimant(ChunkGridEntity, name) if isinstance(name, str) else None ) if entity_type is None: return ChunkGrid.unreadable(shape) diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index fba83d8107..996b98b0d9 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -26,6 +26,7 @@ StorageClass, ValidationProblem, problem, + resolve, ) ACME_DECIMAL_DATA_TYPE_NAME: Final = "acme.decimal" @@ -316,8 +317,10 @@ def test_error_the_constructor_stops_at_the_first_problem_and_coerce_reports_eve with pytest.raises(MetadataValidationError) as caught: AcmeDecimalDataType(AcmeDecimalOptions(precision=0, scale=-1)) assert _locs(caught.value.problems) == [("precision",)] - _, problems = SCOPE.coerce( - DataTypeEntity, {"name": "acme.decimal", "configuration": {"precision": 0, "scale": -1}} + _, problems = resolve( + {"name": "acme.decimal", "configuration": {"precision": 0, "scale": -1}}, + DataTypeEntity, + SCOPE, ) assert _locs(problems) == [("configuration", "precision"), ("configuration", "scale")] diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 8fb5c27dfd..2b81ff939f 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -97,6 +97,7 @@ DataTypeEntity, MetadataEntity, StorageTransformerEntity, + resolve, ) # Every registered entity, keyed by `:` -- an identifier @@ -307,8 +308,8 @@ def test_core_is_a_subset_of_core_and_extensions() -> None: def test_a_name_out_of_scope_resolves_to_nothing() -> None: # Not an error: an unmodelled extension is left unjudged, not rejected. - assert CORE.resolve(CodecEntity, "mycorp.secret") is None - assert CORE.resolve(CodecEntity, "blosc") is BloscCodec + assert CORE.claimant(CodecEntity, "mycorp.secret") is None + assert CORE.claimant(CodecEntity, "blosc") is BloscCodec def test_an_entity_round_trips_through_its_json_form() -> None: @@ -649,27 +650,27 @@ def test_to_json_writes_back_what_was_read(field: type[MetadataEntity], written: # Serialization is not canonicalization. A reader that reads a # document and writes it back must not change bytes it was not asked # to change -- `canonical()` is where you ask. - entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) + entity, problems = resolve(written, field, CORE_AND_EXTENSIONS) assert problems == () assert isinstance(entity, MetadataEntity) assert entity.to_json() == written def test_canonical_is_what_simplifies() -> None: - encoded, _ = CORE_AND_EXTENSIONS.coerce( - ChunkGridEntity, + encoded, _ = resolve( { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": ((32, 32, 32),)}, }, + ChunkGridEntity, + CORE_AND_EXTENSIONS, ) assert isinstance(encoded, MetadataEntity) assert encoded.canonical().to_json() == { "name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": (((32, 3),),)}, } - blosc, _ = CORE_AND_EXTENSIONS.coerce( - CodecEntity, + blosc, _ = resolve( { "name": "blosc", "configuration": { @@ -680,14 +681,15 @@ def test_canonical_is_what_simplifies() -> None: "typesize": 4, }, }, + CodecEntity, + CORE_AND_EXTENSIONS, ) assert isinstance(blosc, BloscCodec) assert "typesize" not in configuration_of(blosc.canonical().to_json()) def test_canonical_reaches_a_contained_entity() -> None: - shard, _ = CORE_AND_EXTENSIONS.coerce( - CodecEntity, + shard, _ = resolve( { "name": "sharding_indexed", "configuration": { @@ -708,6 +710,8 @@ def test_canonical_reaches_a_contained_entity() -> None: "index_codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), }, }, + CodecEntity, + CORE_AND_EXTENSIONS, ) assert isinstance(shard, ShardingIndexedCodec) inner = entry_at(shard.canonical().to_json(), "configuration", "codecs", 1) @@ -717,8 +721,10 @@ def test_canonical_reaches_a_contained_entity() -> None: def test_error_an_explicit_null_scalar_is_refused() -> None: # `null` is a value the document wrote, distinct from absence -- and # no data type admits it as a scalar, so the codec cannot be built. - codec, problems = CORE_AND_EXTENSIONS.coerce( - CodecEntity, {"name": "scale_offset", "configuration": {"offset": None}} + codec, problems = resolve( + {"name": "scale_offset", "configuration": {"offset": None}}, + CodecEntity, + CORE_AND_EXTENSIONS, ) assert codec is not None assert not isinstance(codec, MetadataEntity) @@ -752,7 +758,7 @@ def test_to_json_shares_no_mutable_state_with_the_entity( # The model layer has this test; the entity layer did not, and handed # out its own dict -- so a caller mutating the document it was given # mutated a frozen entity. - entity, problems = CORE_AND_EXTENSIONS.coerce(field, written) + entity, problems = resolve(written, field, CORE_AND_EXTENSIONS) assert problems == () assert isinstance(entity, MetadataEntity) baseline = copy.deepcopy(entity.to_json()) @@ -784,7 +790,7 @@ def test_a_member_the_entity_does_not_model_is_not_written_back() -> None: "typo_key": 1, }, } - codec, problems = CORE_AND_EXTENSIONS.coerce(CodecEntity, entry) + codec, problems = resolve(entry, CodecEntity, CORE_AND_EXTENSIONS) assert [(p.loc, p.kind) for p in problems] == [(("configuration", "typo_key"), "unknown_key")] assert isinstance(codec, BloscCodec) assert "typo_key" not in configuration_of(codec.to_json()) @@ -904,7 +910,7 @@ def test_a_bare_name_entity_holds_the_empty_configuration() -> None: # holds the empty record, reads either spelling, and writes the name. assert Crc32cCodec().configuration == Configuration() for spelling in ("crc32c", {"name": "crc32c"}, {"name": "crc32c", "configuration": {}}): - assert CORE_AND_EXTENSIONS.coerce(CodecEntity, spelling) == (Crc32cCodec(), ()) + assert resolve(spelling, CodecEntity, CORE_AND_EXTENSIONS) == (Crc32cCodec(), ()) assert Crc32cCodec().to_json() == "crc32c" @@ -917,7 +923,7 @@ def test_error_an_entity_of_another_kind_is_invalid_not_out_of_scope() -> None: # `transpose` is in scope, so it is not for another reader to # resolve; it is an array->array codec where a bytes->bytes one goes. value = {"name": "transpose", "configuration": {"order": (0,)}} - entity, problems = CORE_AND_EXTENSIONS.coerce(BytesBytesCodec, value, ("codecs", 2)) + entity, problems = resolve(value, BytesBytesCodec, CORE_AND_EXTENSIONS, ("codecs", 2)) assert entity == Opaque(value, "invalid") assert [(problem.loc, problem.kind, problem.message) for problem in problems] == [ ( diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index f045a96980..985c0fdf42 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -41,6 +41,7 @@ StorageClass, ValidationProblem, problem, + resolve, ) if TYPE_CHECKING: @@ -169,13 +170,15 @@ class Nameless(BytesBytesCodec): def test_the_entity_layer_answers_what_a_reader_needs() -> None: # The questions zarr-python asks before it can read a chunk. - data_type, problems = CORE_AND_EXTENSIONS.coerce(DataTypeEntity, "int32") + data_type, problems = resolve("int32", DataTypeEntity, CORE_AND_EXTENSIONS) assert problems == () assert isinstance(data_type, DataTypeEntity) assert data_type.storage_class() == "multi_byte" - grid, problems = CORE_AND_EXTENSIONS.coerce( - ChunkGridEntity, {"name": "regular", "configuration": {"chunk_shape": (32, 32)}} + grid, problems = resolve( + {"name": "regular", "configuration": {"chunk_shape": (32, 32)}}, + ChunkGridEntity, + CORE_AND_EXTENSIONS, ) assert problems == () assert isinstance(grid, ChunkGridEntity) @@ -202,8 +205,8 @@ class Defaulted(BytesBytesCodec): variable_size: ClassVar[bool] = False assert Defaulted(DefaultedOptions()).configuration.level == 3 - codec, problems = CORE_AND_EXTENSIONS.extended_with(Defaulted).coerce( - CodecEntity, "acme.defaulted" + codec, problems = resolve( + "acme.defaulted", CodecEntity, CORE_AND_EXTENSIONS.extended_with(Defaulted) ) assert problems == () assert isinstance(codec, Defaulted) @@ -319,15 +322,15 @@ def test_a_third_party_can_register_a_family() -> None: # like a single name. scope = CORE_AND_EXTENSIONS.extended_with(AcmeFixedDataType) for name in ("acme.fixed8", "acme.fixed128"): - assert scope.resolve(DataTypeEntity, name) is AcmeFixedDataType - entity, problems = scope.coerce(DataTypeEntity, name) + assert scope.claimant(DataTypeEntity, name) is AcmeFixedDataType + entity, problems = resolve(name, DataTypeEntity, scope) assert problems == () assert isinstance(entity, AcmeFixedDataType) assert entity.to_json() == name # The invented identifier is not a name a document may write, and a # near-miss is still nobody's. - assert scope.resolve(DataTypeEntity, AcmeFixedDataType.identifier) is None - assert scope.resolve(DataTypeEntity, "acme.fixed") is None + assert scope.claimant(DataTypeEntity, AcmeFixedDataType.identifier) is None + assert scope.claimant(DataTypeEntity, "acme.fixed") is None # A rule about the name lands on the field: the document has no # configuration to locate it under. problems = validate_array_metadata_v3( @@ -386,7 +389,7 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: "name": "acme.wrapper", "configuration": {"inner": {"name": "gzip", "configuration": {"level": 5}}}, } - codec, problems = scope.coerce(CodecEntity, entry) + codec, problems = resolve(entry, CodecEntity, scope) assert problems == () assert isinstance(codec, AcmeWrapperCodec) inner = codec.configuration.inner @@ -396,7 +399,7 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: # An inner codec the scope does not model stays verbatim, as anywhere. unknown = {"name": "acme.wrapper", "configuration": {"inner": "acme.unknown"}} - codec, problems = scope.coerce(CodecEntity, unknown) + codec, problems = resolve(unknown, CodecEntity, scope) assert problems == () assert isinstance(codec, AcmeWrapperCodec) assert isinstance(codec.configuration.inner, Opaque) @@ -407,7 +410,7 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: "name": "acme.wrapper", "configuration": {"inner": {"name": "gzip", "configuration": {"level": 99}}}, } - _, problems = scope.coerce(CodecEntity, bad) + _, problems = resolve(bad, CodecEntity, scope) assert [problem.loc for problem in problems] == [ ("configuration", "inner", "configuration", "level") ] @@ -428,7 +431,7 @@ def test_a_third_party_entity_containing_entities_reads_them_in_scope() -> None: } }, } - codec, _ = scope.coerce(CodecEntity, verbose) + codec, _ = resolve(verbose, CodecEntity, scope) assert isinstance(codec, AcmeWrapperCodec) inner = codec.canonical().configuration.inner assert isinstance(inner, BloscCodec) @@ -541,12 +544,12 @@ def test_a_rule_about_a_member_is_the_record_s_own() -> None: # configuration; `coerce` runs it to the end and locates what it # yields in the document; the constructor stops at the first. scope = CORE_AND_EXTENSIONS.extended_with(AcmeBlockCodec) - codec, problems = scope.coerce( - CodecEntity, {"name": "acme.block", "configuration": {"block": 64}} + codec, problems = resolve( + {"name": "acme.block", "configuration": {"block": 64}}, CodecEntity, scope ) assert problems == () assert isinstance(codec, AcmeBlockCodec) - _, problems = scope.coerce(CodecEntity, {"name": "acme.block", "configuration": {"block": 6}}) + _, problems = resolve({"name": "acme.block", "configuration": {"block": 6}}, CodecEntity, scope) assert [(p.loc, p.message) for p in problems] == [ (("configuration", "block"), "expected a power of two, got 6") ] @@ -555,7 +558,9 @@ def test_a_rule_about_a_member_is_the_record_s_own() -> None: AcmeBlockCodec(AcmeBlockOptions(block=6)) assert [p.loc for p in caught.value.problems] == [("block",)] # A member that failed its type check never reaches the rule. - _, problems = scope.coerce(CodecEntity, {"name": "acme.block", "configuration": {"block": "x"}}) + _, problems = resolve( + {"name": "acme.block", "configuration": {"block": "x"}}, CodecEntity, scope + ) assert [p.kind for p in problems] == ["invalid_type"] @@ -594,8 +599,8 @@ def test_the_constructor_stops_at_the_first_problem_and_coerce_reports_every_one AcmeRangeCodec(AcmeRangeOptions(low=-1, high=-2)) assert [p.loc for p in caught.value.problems] == [("low",)] scope = CORE_AND_EXTENSIONS.extended_with(AcmeRangeCodec) - _, problems = scope.coerce( - CodecEntity, {"name": "acme.range", "configuration": {"low": -1, "high": -2}} + _, problems = resolve( + {"name": "acme.range", "configuration": {"low": -1, "high": -2}}, CodecEntity, scope ) assert [p.loc for p in problems] == [("configuration", "low"), ("configuration", "high")] assert list(AcmeRangeOptions(low=0, high=1).problems()) == [] @@ -662,10 +667,10 @@ def test_a_reader_gets_structural_and_semantic_reasons_together() -> None: def test_a_malformed_envelope_is_one_problem() -> None: # The envelope is judged once, by the scope; the entity is not asked # to read what is not a metadata field. - _, problems = CORE_AND_EXTENSIONS.coerce(CodecEntity, 5, ("codecs", 0)) + _, problems = resolve(5, CodecEntity, CORE_AND_EXTENSIONS, ("codecs", 0)) assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0), "invalid_type")] - _, problems = CORE_AND_EXTENSIONS.coerce( - CodecEntity, {"name": "gzip", "configuration": 42}, ("codecs", 0) + _, problems = resolve( + {"name": "gzip", "configuration": 42}, CodecEntity, CORE_AND_EXTENSIONS, ("codecs", 0) ) assert [(p.loc, p.kind) for p in problems] == [(("codecs", 0, "configuration"), "invalid_type")] @@ -725,13 +730,13 @@ def transition(self, incoming: ArrayParts) -> ArrayParts | None: scope = CORE_AND_EXTENSIONS.extended_with(AcmeScaled) for spelled in (2, 2.5): - codec, problems = scope.coerce( - CodecEntity, {"name": "acme.scaled", "configuration": {"scale": spelled}} + codec, problems = resolve( + {"name": "acme.scaled", "configuration": {"scale": spelled}}, CodecEntity, scope ) assert problems == () assert isinstance(codec, AcmeScaled) - _, problems = scope.coerce( - CodecEntity, {"name": "acme.scaled", "configuration": {"scale": True}} + _, problems = resolve( + {"name": "acme.scaled", "configuration": {"scale": True}}, CodecEntity, scope ) assert [(p.loc, p.message) for p in problems] == [ (("configuration", "scale"), "expected a number, got True") diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py index 89a3ee521c..75e82c68cf 100644 --- a/packages/zarr-metadata/tests/v3/test_fill_values.py +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -11,7 +11,7 @@ from tests.helpers import entry_at from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS -from zarr_metadata.v3.entity import DataTypeEntity +from zarr_metadata.v3.entity import DataTypeEntity, resolve # (data type metadata, a fill value it accepts) ACCEPTED: dict[str, tuple[object, object]] = { @@ -75,7 +75,7 @@ def _data_type(metadata: object) -> DataTypeEntity: name = metadata if isinstance(metadata, str) else entry_at(metadata, "name") assert isinstance(name, str), metadata - entity_type = CORE_AND_EXTENSIONS.resolve(DataTypeEntity, name) + entity_type = CORE_AND_EXTENSIONS.claimant(DataTypeEntity, name) assert entity_type is not None, metadata entity, problems = entity_type.coerce(metadata, CORE_AND_EXTENSIONS) assert problems == (), problems @@ -98,7 +98,7 @@ def test_error_rejects(metadata: object, fill: object, reason: str) -> None: def test_error_a_malformed_raw_name_has_no_entity_to_ask() -> None: # `r12` is not a width, so the data type does not exist and there is # nothing to put a fill value to. - entity, problems = CORE_AND_EXTENSIONS.coerce(DataTypeEntity, "r12") + entity, problems = resolve("r12", DataTypeEntity, CORE_AND_EXTENSIONS) assert not isinstance(entity, DataTypeEntity) assert [problem.message for problem in problems] == [ "Expected 'r' where N is a positive multiple of 8, got 'r12'" @@ -107,4 +107,4 @@ def test_error_a_malformed_raw_name_has_no_entity_to_ask() -> None: def test_an_unmodelled_data_type_judges_nothing() -> None: # Extension openness: a fill value we cannot interpret is not wrong. - assert CORE_AND_EXTENSIONS.resolve(DataTypeEntity, "mycorp.decimal") is None + assert CORE_AND_EXTENSIONS.claimant(DataTypeEntity, "mycorp.decimal") is None diff --git a/packages/zarr-metadata/tests/v3/test_resolve.py b/packages/zarr-metadata/tests/v3/test_resolve.py index 05bfdc3a1b..83d9f533b2 100644 --- a/packages/zarr-metadata/tests/v3/test_resolve.py +++ b/packages/zarr-metadata/tests/v3/test_resolve.py @@ -50,7 +50,7 @@ def test_a_name_resolves_to_the_entity_that_answers_for_it( field: type[MetadataEntity], name: str, expected: type[MetadataEntity] | None ) -> None: - assert CORE_AND_EXTENSIONS.resolve(field, name) is expected + assert CORE_AND_EXTENSIONS.claimant(field, name) is expected @given(width=st.integers(min_value=0, max_value=2**32)) @@ -59,7 +59,7 @@ def test_every_numeric_r_spelling_resolves_to_the_family(width: int) -> None: # the family claims a name by grammar shape, not by validity, so a # misspelled member of a family we model is reported as a misspelling # rather than passing as an unknown third-party extension. - assert CORE_AND_EXTENSIONS.resolve(DataTypeEntity, f"r{width}") is RawBytesDataType + assert CORE_AND_EXTENSIONS.claimant(DataTypeEntity, f"r{width}") is RawBytesDataType OTHER_KINDS: tuple[type[MetadataEntity], ...] = (CodecEntity, ChunkGridEntity) @@ -71,7 +71,7 @@ def test_r_shaped_names_resolve_to_nothing_outside_data_types( ) -> None: # The family belongs to `data_type`; a codec that happens to be named # `r8` must not reach it. - assert CORE_AND_EXTENSIONS.resolve(field, f"r{width}") is None + assert CORE_AND_EXTENSIONS.claimant(field, f"r{width}") is None # The scan `resolve` falls back to asks every entity, so a name no entity @@ -86,7 +86,7 @@ def test_r_shaped_names_resolve_to_nothing_outside_data_types( @given(name=_UNCLAIMED) def test_a_name_no_entity_claims_resolves_to_nothing(name: str) -> None: - assert CORE_AND_EXTENSIONS.resolve(DataTypeEntity, name) is None + assert CORE_AND_EXTENSIONS.claimant(DataTypeEntity, name) is None def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: From 9c844e2660662808d889bff85b6a353453e3dd0e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 11:25:05 +0200 Subject: [PATCH 102/107] feat(zarr-metadata): three layers of reading, and the third produces the resolved pipeline Reading a v3 array document is now three explicit layers, ordered by what each needs and each handing the next a typed value with its problems. `well_formed_array_v3(value)` needs only the value: `refine_json` turns it into JSON with arrays as tuples, string keys and finite floats in one walk, or says at which leaves it is not, and the model layer judges the document's shape. Nothing downstream normalizes or checks JSON-ness again: the `envelope_judged` flag and the second normalization inside `coerce` are gone, `Opaque.json` is `JSONValue` and checked, and the class routine takes refined JSON. `read_array_v3(document, context)` needs a scope: each extension point is handed to `read_field`, which is what `resolve` does once a field's envelope is judged. `resolve(data, kind, context)` is the first two layers for a field on its own. `refine_array_v3(array)` needs the array, and returns `RefinedArrayV3`: the `ArrayParts` the pipeline is handed and a `Pipeline` of `PipelineStage`s, each a codec and the array that reaches it, a shard's `codecs` and `index_codecs` refined inside its stage. This is what a codec pipeline is built from; validating the composition is what the walk finds on the way. A codec that holds pipelines declares them through `inner_pipelines(incoming)` and judges nothing inside them itself, so the sharding codec's own walk is gone with `chain_problems`. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems, 0 crashes. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../zarr-metadata/changes/4379.feature.11.md | 28 ++ .../src/zarr_metadata/model/_validation.py | 48 +++ .../src/zarr_metadata/rules/_documents.py | 118 ++++--- .../src/zarr_metadata/v3/_chain.py | 63 +++- .../src/zarr_metadata/v3/_document.py | 292 +++++++++++------- .../src/zarr_metadata/v3/_entity.py | 163 ++++++---- .../src/zarr_metadata/v3/_typed_json.py | 18 -- .../v3/codec/sharding_indexed.py | 66 ++-- .../src/zarr_metadata/v3/entity.py | 43 ++- .../tests/rules/test_chunk_grid.py | 25 +- .../zarr-metadata/tests/test_public_api.py | 3 + .../zarr-metadata/tests/v3/test_entities.py | 119 ++++++- .../tests/v3/test_extension_api.py | 3 +- .../tests/v3/test_fill_values.py | 9 +- 14 files changed, 667 insertions(+), 331 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.feature.11.md diff --git a/packages/zarr-metadata/changes/4379.feature.11.md b/packages/zarr-metadata/changes/4379.feature.11.md new file mode 100644 index 0000000000..7368ca5731 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.feature.11.md @@ -0,0 +1,28 @@ +Reading a v3 array document is three layers, ordered by what each needs, +each handing the next a typed value and its problems. `well_formed_array_v3(value)` +needs only the value: the JSON is refined -- arrays as tuples, string +keys, finite floats -- and the document's shape judged by the model layer; +a value that is not JSON gets that verdict and nothing else is judged. +`read_array_v3(document, context)` needs a scope: each extension point's +name is related to a class through the context and the class is handed +the field. `refine_array_v3(array)` needs the array: the fill value +against its type, the grid against the shape, and the codec pipeline +walked with what reaches each codec. + +The third layer's value is the resolved pipeline. `RefinedArrayV3` +carries the `ArrayParts` the pipeline is handed and a `Pipeline` of +`PipelineStage`s, each a codec and the array that reaches it -- None past +the array->bytes boundary or after a codec that could not say what it +does -- with a shard's `codecs` and `index_codecs` refined inside its +stage. This is what a codec pipeline is built from, and what zarr-python +computes separately today when it evolves an array spec codec by codec; +validating the composition is what the walk finds on the way. A codec +that holds pipelines declares them through `inner_pipelines(incoming)`, +each by the member that holds it with the parts it is handed, and +judges nothing inside them itself. + +`resolve(data, kind, context)` is the first two layers for a field on +its own. Nothing downstream of the first layer normalizes or checks +JSON-ness again, so the reader no longer carries an `envelope_judged` +flag or re-normalizes arrays inside `coerce`; `chain_problems` is gone +from the door, replaced by the refinement that produces the pipeline. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 58b52158b4..b80ab6f6e7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -134,6 +134,54 @@ def validate_json(value: object) -> tuple[ValidationProblem, ...]: return (ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type"),) +def refine_json( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """`value` as JSON with arrays as tuples, or None with every reason it is not JSON. + + The first layer of reading, which needs nothing but the value. One + walk normalizes and judges: a mapping becomes a `dict` with string + keys, a sequence a tuple, a float must be finite. Everything after + it takes `JSONValue` and normalizes nothing. A value that is not + JSON is None, with the problems located at the leaves that are not: + not JSON is the first verdict, and there is nothing to read. + """ + if isinstance(value, float): + if math.isfinite(value): + return value, () + return None, ( + ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value"), + ) + if isinstance(value, (str, int, bool)) or value is None: + return value, () + if isinstance(value, Mapping): + members: dict[str, JSONValue] = {} + problems: list[ValidationProblem] = [] + for key, item in cast("Mapping[object, object]", value).items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") + ) + continue + member, found = refine_json(item, (*loc, key)) + problems.extend(found) + if len(found) == 0: + members[key] = member + return (members if len(problems) == 0 else None), tuple(problems) + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): + entries: list[JSONValue] = [] + found_in_entries: list[ValidationProblem] = [] + for index, item in enumerate(cast("Sequence[object]", value)): + entry, found = refine_json(item, (*loc, index)) + found_in_entries.extend(found) + if len(found) == 0: + entries.append(entry) + return (tuple(entries) if len(found_in_entries) == 0 else None), tuple(found_in_entries) + return None, ( + ValidationProblem(loc, f"not a JSON-serializable value: {value!r}", "invalid_type"), + ) + + def _is_canonical_json(value: object) -> TypeIs[JSONValue]: """Whether `value` already uses the concrete containers in `JSONValue`.""" if isinstance(value, float): diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index e6e23d07d2..2fc90caa56 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -22,14 +22,11 @@ from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._validation import ( MetadataValidationError, - arrays_to_tuples, + refine_json, ) from zarr_metadata.model._validation import ( validate_array_metadata_v2 as _validate_structure_v2, ) -from zarr_metadata.model._validation import ( - validate_array_metadata_v3 as _validate_structure_v3, -) from zarr_metadata.model._validation import ( validate_group_metadata_v2 as _validate_group_structure_v2, ) @@ -37,12 +34,18 @@ validate_group_metadata_v3 as _validate_group_structure_v3, ) from zarr_metadata.v2._document import array_problems_v2 -from zarr_metadata.v3._document import array_problems_v3, group_problems_v3, read_array_v3 +from zarr_metadata.v3._document import ( + group_problems_v3, + read_array_v3, + refine_array_v3, + well_formed_array_v3, +) from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS, Context if TYPE_CHECKING: from collections.abc import Callable + from zarr_metadata._common import JSONValue from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v2.array import ZarrV2ArrayMetadataJSON from zarr_metadata.v2.group import ZarrV2GroupMetadataJSON @@ -50,7 +53,7 @@ from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON _StructuralValidator = Callable[[object], tuple[ValidationProblem, ...]] - _SemanticValidator = Callable[[Mapping[str, object]], tuple[ValidationProblem, ...]] + _SemanticValidator = Callable[[Mapping[str, JSONValue]], tuple[ValidationProblem, ...]] DocumentT = TypeVar("DocumentT") @@ -76,25 +79,11 @@ def __post_init__(self) -> None: raise ValueError(msg) -def _no_semantics(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: +def _no_semantics(document: Mapping[str, JSONValue]) -> tuple[ValidationProblem, ...]: """v2 group documents carry no cross-field constraints.""" return () -def _array_semantics_v3(context: Context) -> _SemanticValidator: - """The v3 array semantics, asked in `context`. - - The work is `zarr_metadata.v3` asking each entity about itself and - about the parts of the document it meets; what this layer decides is - which entities are in scope while it asks. - """ - - def judge(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: - return array_problems_v3(document, context) - - return judge - - def _group_semantics_v3(context: Context) -> _SemanticValidator: """The v3 group semantics, asked in `context`. @@ -102,24 +91,28 @@ def _group_semantics_v3(context: Context) -> _SemanticValidator: and those are array and group documents judged in the same scope. """ - def judge(document: Mapping[str, object]) -> tuple[ValidationProblem, ...]: + def judge(document: Mapping[str, JSONValue]) -> tuple[ValidationProblem, ...]: return group_problems_v3(document, context) return judge def _judged( - normalized: object, structure: _StructuralValidator, semantics: _SemanticValidator -) -> tuple[ValidationProblem, ...]: - """Structural and semantic problems in an already-normalized document. + value: object, structure: _StructuralValidator, semantics: _SemanticValidator +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """`value` refined to JSON, and its structural and semantic problems. - Takes the normalized value rather than the caller's input so that - `validate_*` and `parse_*` each walk the document once. + The layers for a document without a codec pipeline: JSON first, + then the shape, then whatever the semantics need of an object. A + value that is not JSON is None with only that verdict. """ - problems = structure(normalized) - if isinstance(normalized, Mapping): - problems = problems + semantics(cast("Mapping[str, object]", normalized)) - return tuple(problems) + refined, problems = refine_json(value) + if refined is None: + return None, problems + problems = structure(refined) + if isinstance(refined, Mapping): + problems = (*problems, *semantics(cast("Mapping[str, JSONValue]", refined))) + return refined, tuple(problems) def validate_array_metadata_v3( @@ -134,13 +127,20 @@ def validate_array_metadata_v3( that could not be built -- so a document with two defects in one configuration may need a second pass. The verdict is never affected. - Structural problems (from the model layer) and semantic problems - (from the entities themselves) are reported together. JSON arrays are - normalized to tuples before judgment, so list-spelled documents - (e.g. fresh `json.loads` output) are judged at the canonical data - level rather than rejected for their spelling. + The three layers of reading, in order, each handing the next what it + needs: the value refined to JSON and judged for shape, the extension + points read in `context`, the whole refined against the array. Their + problems are reported together; a value that is not JSON gets only + that verdict. List-spelled documents (fresh `json.loads` output) are + judged at the canonical data level rather than rejected for their + spelling. """ - return _judged(arrays_to_tuples(value), _validate_structure_v3, _array_semantics_v3(context)) + document, problems = well_formed_array_v3(value) + if document is None: + return problems + array, found = read_array_v3(document, context) + _, composed = refine_array_v3(array) + return (*problems, *found, *composed) def parse_array_metadata_v3( @@ -152,11 +152,14 @@ def parse_array_metadata_v3( `MetadataValidationError` carrying every structural and composition problem found. """ - normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_structure_v3, _array_semantics_v3(context)) - if len(problems) != 0: + document, problems = well_formed_array_v3(value) + if document is not None: + array, found = read_array_v3(document, context) + _, composed = refine_array_v3(array) + problems = (*problems, *found, *composed) + if document is None or len(problems) != 0: raise MetadataValidationError(problems) - return cast("ZarrV3ArrayMetadataJSON", normalized) + return cast("ZarrV3ArrayMetadataJSON", document) def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: @@ -165,7 +168,7 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: JSON arrays are normalized to tuples before judgment, as in `validate_array_metadata_v3`. """ - return _judged(arrays_to_tuples(value), _validate_structure_v2, array_problems_v2) + return _judged(value, _validate_structure_v2, array_problems_v2)[1] def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: @@ -175,11 +178,10 @@ def parse_array_metadata_v2(value: object) -> ZarrV2ArrayMetadataJSON: `MetadataValidationError` carrying every structural and composition problem found. """ - normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_structure_v2, array_problems_v2) + refined, problems = _judged(value, _validate_structure_v2, array_problems_v2) if len(problems) != 0: raise MetadataValidationError(problems) - return cast("ZarrV2ArrayMetadataJSON", normalized) + return cast("ZarrV2ArrayMetadataJSON", refined) def validate_group_metadata_v3( @@ -191,20 +193,17 @@ def validate_group_metadata_v3( consolidated child document invalid under its own rules is reported here, at its path. """ - return _judged( - arrays_to_tuples(value), _validate_group_structure_v3, _group_semantics_v3(context) - ) + return _judged(value, _validate_group_structure_v3, _group_semantics_v3(context))[1] def parse_group_metadata_v3( value: object, *, context: Context = CORE_AND_EXTENSIONS ) -> ZarrV3GroupMetadataJSON: """Return `value` as a valid `ZarrV3GroupMetadataJSON`, or raise.""" - normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_group_structure_v3, _group_semantics_v3(context)) + refined, problems = _judged(value, _validate_group_structure_v3, _group_semantics_v3(context)) if len(problems) != 0: raise MetadataValidationError(problems) - return cast("ZarrV3GroupMetadataJSON", normalized) + return cast("ZarrV3GroupMetadataJSON", refined) def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: @@ -213,16 +212,15 @@ def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: v2 group documents carry no composition constraints today, so this is the structural judgment, offered here for a uniform read-side API. """ - return _judged(arrays_to_tuples(value), _validate_group_structure_v2, _no_semantics) + return _judged(value, _validate_group_structure_v2, _no_semantics)[1] def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: """Return `value` as a valid `ZarrV2GroupMetadataJSON`, or raise.""" - normalized = arrays_to_tuples(value) - problems = _judged(normalized, _validate_group_structure_v2, _no_semantics) + refined, problems = _judged(value, _validate_group_structure_v2, _no_semantics) if len(problems) != 0: raise MetadataValidationError(problems) - return cast("ZarrV2GroupMetadataJSON", normalized) + return cast("ZarrV2GroupMetadataJSON", refined) def canonicalize_array_metadata_v3( @@ -252,12 +250,12 @@ def canonicalize_array_metadata_v3( is a bug in the entity, not a verdict on the document, which was valid. """ - normalized = arrays_to_tuples(document) - problems = _validate_structure_v3(normalized) - if not isinstance(normalized, Mapping): + refined, problems = well_formed_array_v3(document) + if refined is None: return Invalid(problems) - array, found = read_array_v3(cast("Mapping[str, object]", normalized), context) - problems = (*problems, *found, *array.problems()) + array, found = read_array_v3(refined, context) + _, composed = refine_array_v3(array) + problems = (*problems, *found, *composed) if len(problems) != 0: return Invalid(problems) return Canonical(ZarrV3ArrayMetadata.from_json(array.canonical().to_json()).to_json()) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py index aa926e68a1..2281e94889 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_chain.py @@ -7,9 +7,10 @@ `cast_value` changes the element type, and a shard that follows either one sees the transformed array. -This is where the walk lives rather than in `zarr_metadata.rules`, -because a `sharding_indexed` codec holds two pipelines of its own and has -to walk them to judge itself. +The walk produces the resolved pipeline: at each position the codec and +the array that reaches it, and for a codec that holds pipelines of its +own -- a shard's inner chunks and its index -- those pipelines refined +the same way. Validation is what the walk finds on the way. Propagation stops -- every later codec receives `None` -- at the array-to-bytes boundary, where there is no array left, and after any @@ -20,18 +21,41 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._entity import ArrayArrayCodec, ArrayBytesCodec, CodecEntity, within if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Mapping, Sequence from zarr_metadata.v3._entity import Loc, Opaque from zarr_metadata.v3._parts import ArrayParts +@dataclass(frozen=True, slots=True) +class PipelineStage: + """One position of a refined pipeline: the codec, and the array that reaches it.""" + + codec: CodecEntity | Opaque + incoming: ArrayParts | None + """What reaches this codec. + + None past the array->bytes boundary, where there is no array, and + after a codec that could not say what it does to one. + """ + inner: Mapping[str, Pipeline] + """The pipelines this codec holds, refined, by the member that holds each; empty for most.""" + + +@dataclass(frozen=True, slots=True) +class Pipeline: + """A codec pipeline refined against the array handed to it: what reaches each codec.""" + + stages: tuple[PipelineStage, ...] + + def _stage(codec: CodecEntity) -> tuple[int, str]: """Where in the pipeline a codec stands, as a rank and as the spec names it.""" if isinstance(codec, ArrayArrayCodec): @@ -44,7 +68,7 @@ def _stage(codec: CodecEntity) -> tuple[int, str]: def _label(codec: CodecEntity | Opaque) -> str: if isinstance(codec, CodecEntity): return repr(type(codec).identifier) - return repr(codec) + return repr(codec.json) def order_problems( @@ -91,31 +115,44 @@ def order_problems( return tuple(problems) -def chain_problems( +def refine_pipeline( codecs: Sequence[CodecEntity | Opaque], start: ArrayParts | None, loc: Loc -) -> tuple[ValidationProblem, ...]: - """Every problem this pipeline has, ordering and per-codec alike. - - `start` is what the first codec receives: the document's own array, or - a shard's inner chunk, or its index. +) -> tuple[Pipeline, tuple[ValidationProblem, ...]]: + """The pipeline refined against `start`, with every problem found on the way. + + `start` is what the first codec receives: the document's own array, + or a shard's inner chunk, or its index. Each codec is asked what it + cannot take of what reaches it, then what it hands on; a codec that + holds pipelines has them refined in turn, located under the member + that holds each. Ordering is judged first, over the whole pipeline. """ problems = list(order_problems(codecs, loc)) + stages: list[PipelineStage] = [] incoming = start for index, codec in enumerate(codecs): if not isinstance(codec, CodecEntity): # Out of scope: unjudged, and everything after it is too. + stages.append(PipelineStage(codec, incoming, {})) incoming = None continue problems.extend(within((*loc, index), codec.incoming_problems(incoming))) + inner: dict[str, Pipeline] = {} + for member, (held, handed) in codec.inner_pipelines(incoming).items(): + pipeline, found = refine_pipeline(held, handed, (*loc, index, "configuration", member)) + inner[member] = pipeline + problems.extend(found) + stages.append(PipelineStage(codec, incoming, inner)) incoming = ( codec.transition(incoming) if incoming is not None and isinstance(codec, ArrayArrayCodec) else None ) - return tuple(problems) + return Pipeline(tuple(stages)), tuple(problems) __all__ = [ - "chain_problems", + "Pipeline", + "PipelineStage", "order_problems", + "refine_pipeline", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index d59738cb5d..62692ce418 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -1,17 +1,23 @@ -"""A whole v3 array document, read as entities and judged as a whole. - -The document-level check is a composition of the entities' own checks, -not a second implementation of them. It does three things in order: - -1. read every extension point in a scope, which is type-space; -2. ask each entity what is wrong with its own values; -3. ask the questions that span fields -- the fill value against the data - type, the grid against the shape, the pipeline against the array -- - each by handing an entity the part of the document it needs. - -Nothing here knows what `blosc` or `int32` or `rectilinear` is. A new -extension is a class and a registry entry, and this module does not -change. +"""A whole v3 array document, read in three layers, each with what it needs. + +1. `well_formed_array_v3`: the value alone. JSON syntax and the + document's shape -- arrays as tuples, string keys, finite floats, the + keys a v3 array has and the shapes their values take, the envelope of + each extension point. +2. `read_array_v3`: a scope. Each extension point's name related to a + class in the `Context`, and the class handed the field: the + configuration parsed against its record, the rules asked, nested + entities read the same way. +3. `refine_array_v3`: the array. The fill value against the data type, + the grid against the shape, and the codec pipeline walked with what + reaches each codec, which is the resolved pipeline -- the array each + codec is handed, and a shard's inner pipelines refined the same way. + Validation is what the walk finds. + +Each layer hands the next a typed value and its problems; the next reads +what it can and never repeats the work of the one before. Nothing here +knows what `blosc` or `int32` or `rectilinear` is. A new extension is a +class and a registry entry, and this module does not change. """ from __future__ import annotations @@ -23,12 +29,12 @@ from zarr_metadata.model._validation import ( MetadataValidationError, ValidationProblem, - arrays_to_tuples, + refine_json, ) from zarr_metadata.model._validation import ( validate_array_metadata_v3 as validate_array_metadata_v3_structure, ) -from zarr_metadata.v3._chain import chain_problems +from zarr_metadata.v3._chain import Pipeline, refine_pipeline from zarr_metadata.v3._entity import ( ChunkGridEntity, ChunkKeyEncodingEntity, @@ -39,7 +45,7 @@ StorageTransformerEntity, held_problems, problem, - resolve, + read_field, within, ) from zarr_metadata.v3._parts import ArrayParts, ChunkGrid @@ -48,6 +54,7 @@ if TYPE_CHECKING: from collections.abc import Sequence + from zarr_metadata._common import JSONValue from zarr_metadata.v3._entity import Loc @@ -56,16 +63,16 @@ @dataclass(frozen=True, slots=True) class ArrayDocumentV3: - """A v3 array document with its extension points read as entities. + """A v3 array document with its extension points read as entities: the second layer's value. A field that could not be read holds an `Opaque`, which carries the JSON the document wrote and says whether the name was out of scope -- an extension this reader does not model, which is not an error -- or claimed and refused. Both are narrowable: every field is an exhaustive - two-case union. + two-case union. `refine_array_v3` takes it on to the third layer. """ - document: Mapping[str, object] + document: Mapping[str, JSONValue] data_type: DataTypeEntity | Opaque chunk_grid: ChunkGridEntity | Opaque chunk_key_encoding: ChunkKeyEncodingEntity | Opaque @@ -102,27 +109,6 @@ def __post_init__(self) -> None: if len(found) != 0: raise MetadataValidationError(found) - def problems(self) -> tuple[ValidationProblem, ...]: - """Every semantic problem this document has, once it has been read. - - The type-space problems are `read_array_v3`'s, because they are - the reasons some of this is `Opaque` rather than an entity. - """ - # No per-entity value problems: an entity exists only if its own - # values are allowed, so `read_array_v3` has already reported any. - # A pipeline the document did not write as an array was not read, - # and an empty one would be judged for a verdict about nothing. - return ( - *_fill_value_problems(self), - *_grid_problems(self), - *_dimension_names_problems(self), - *( - chain_problems(self.codecs, self.parts, ("codecs",)) - if _listed(self.document, "codecs") is not None - else () - ), - ) - def canonical(self) -> ArrayDocumentV3: """This document in the simplest form that means the same thing. @@ -151,7 +137,7 @@ def canonical(self) -> ArrayDocumentV3: del document["dimension_names"] return replace(simplified, document=document) - def to_json(self) -> dict[str, object]: + def to_json(self) -> dict[str, JSONValue]: """The document as it was written, with each entity's members as the entity has them. Faithful: a document read and written comes out as it went in, @@ -169,7 +155,7 @@ def to_json(self) -> dict[str, object]: return { **self.document, **{ - key: _as_written(self.document[key], value) + key: cast("JSONValue", _as_written(self.document[key], value)) for key, value in _rendered(self).items() }, } @@ -179,10 +165,12 @@ def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> """A v3 array document read into entities, or raise. The reader's front door, and the one entry point that fails fast: - one call, and either every extension point is read or a single - `MetadataValidationError` carries every reason it is not -- - structural and semantic together. Use `validate_array_metadata_v3` - instead when you want the problems as data. + all three layers, and either every extension point is read and + the whole composes, or a single `MetadataValidationError` carries + every reason it does not -- structural and semantic together. Use + `validate_array_metadata_v3` instead when you want the problems + as data, and the layers themselves when you want to stop between + them. A name this `context` does not model is *not* a failure. It comes back as an `Opaque` marked `out_of_scope`, because a document may @@ -190,34 +178,128 @@ def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> refusing it would make openness unimplementable. What fails is metadata that is wrong, not metadata that is unfamiliar. """ - normalized = arrays_to_tuples(value) - problems = validate_array_metadata_v3_structure(normalized) - if isinstance(normalized, Mapping): - # Read whatever the structure allowed, so a structural - # problem does not hide the semantic ones behind it. - array, found = read_array_v3(cast("Mapping[str, object]", normalized), context) - problems = (*problems, *found, *array.problems()) + document, problems = well_formed_array_v3(value) + if document is not None: + # Read whatever the shape allowed, so a structural problem does + # not hide the semantic ones behind it. + array, found = read_array_v3(document, context) + _, composed = refine_array_v3(array) + problems = (*problems, *found, *composed) if len(problems) == 0: return array - if len(problems) == 0: # pragma: no cover - a non-mapping always has problems - problems = (ValidationProblem((), "expected a v3 array document", "invalid_type"),) raise MetadataValidationError(problems) - @property - def parts(self) -> ArrayParts: - """The array the codec pipeline is handed.""" - shape = self.document.get("shape") - # A grid out of scope still divides an array of some rank, and the - # shape is what pins it -- which is enough to catch a shard whose - # inner chunk has the wrong number of dimensions. - grid = ( - self.chunk_grid.grid(shape) - if isinstance(self.chunk_grid, ChunkGridEntity) - else ChunkGrid.unreadable(shape) - ) - return ArrayParts( - grid, self.data_type if isinstance(self.data_type, DataTypeEntity) else None - ) + +@dataclass(frozen=True, slots=True) +class RefinedArrayV3: + """A v3 array document refined against its own array: the third layer's value. + + `parts` is the array the codec pipeline is handed -- its chunks, + under its grid, of its data type -- and `pipeline` is that pipeline + resolved: at each position the codec and what reaches it, a shard's + inner pipelines refined inside it. What a codec pipeline is built + from, and what validating the composition finds on the way. + """ + + array: ArrayDocumentV3 + parts: ArrayParts + pipeline: Pipeline + + +def well_formed_array_v3( + value: object, +) -> tuple[Mapping[str, JSONValue] | None, tuple[ValidationProblem, ...]]: + """The first layer: `value` as a refined v3 array document, with every structural problem. + + Needs nothing but the value. The JSON is refined -- arrays as + tuples, string keys, finite floats -- and the document's shape is + judged by the model layer: the keys a v3 array has, the shapes their + values take, the envelope of each extension point. What comes back + is refined JSON that the next layer reads without normalizing or + judging JSON-ness again, and the structural problems beside it, which + do not stop the next layer from reading what it can. A value that is + not JSON, or not an object, is None with the reasons: not JSON is + the first verdict, and there is nothing to read. + """ + refined, problems = refine_json(value) + if refined is None: + return None, problems + if not isinstance(refined, Mapping): + return None, problem((), f"expected a v3 array document as an object, got {refined!r}") + document = cast("Mapping[str, JSONValue]", refined) + return document, validate_array_metadata_v3_structure(document) + + +def read_array_v3( + document: Mapping[str, JSONValue], context: Context +) -> tuple[ArrayDocumentV3, tuple[ValidationProblem, ...]]: + """The second layer: `document`'s extension points, read in `context`. + + Needs a scope. The one place that knows which of a document's fields + holds which kind of entity; each is handed to `read_field`, its + envelope having been judged with the document. Type-space only: what + comes back is well-typed by construction, and the problems are the + reasons some of it is not an entity. + """ + data_type, found_1 = _read_one(context, DataTypeEntity, document, "data_type") + chunk_grid, found_2 = _read_one(context, ChunkGridEntity, document, "chunk_grid") + encoding, found_3 = _read_one(context, ChunkKeyEncodingEntity, document, "chunk_key_encoding") + codecs, found_4 = _read_each(context, CodecEntity, document, "codecs") + transformers, found_5 = _read_each( + context, StorageTransformerEntity, document, "storage_transformers" + ) + return ( + ArrayDocumentV3( + document=document, + data_type=data_type, + chunk_grid=chunk_grid, + chunk_key_encoding=encoding, + codecs=codecs, + storage_transformers=transformers, + ), + (*found_1, *found_2, *found_3, *found_4, *found_5), + ) + + +def refine_array_v3(array: ArrayDocumentV3) -> tuple[RefinedArrayV3, tuple[ValidationProblem, ...]]: + """The third layer: `array` against its own array, and the pipeline resolved. + + Needs the array: the fill value is judged by the data type it fills, + the grid against the shape it divides, the dimension names counted + against it, and the codec pipeline walked from the parts the grid + and data type make, each codec handed what reaches it. A pipeline + the document did not write as an array was not read, and is not + judged as an empty one. + """ + parts = _parts(array) + pipeline, composed = ( + refine_pipeline(array.codecs, parts, ("codecs",)) + if _listed(array.document, "codecs") is not None + else (Pipeline(()), ()) + ) + problems = ( + *_fill_value_problems(array), + *_grid_problems(array), + *_dimension_names_problems(array), + *composed, + ) + return RefinedArrayV3(array, parts, pipeline), problems + + +def _parts(array: ArrayDocumentV3) -> ArrayParts: + """The array the codec pipeline is handed.""" + shape = array.document.get("shape") + # A grid out of scope still divides an array of some rank, and the + # shape is what pins it -- which is enough to catch a shard whose + # inner chunk has the wrong number of dimensions. + grid = ( + array.chunk_grid.grid(shape) + if isinstance(array.chunk_grid, ChunkGridEntity) + else ChunkGrid.unreadable(shape) + ) + return ArrayParts( + grid, array.data_type if isinstance(array.data_type, DataTypeEntity) else None + ) def _as_object(value: object) -> tuple[ValidationProblem, ...]: @@ -227,9 +309,9 @@ def _as_object(value: object) -> tuple[ValidationProblem, ...]: return problem((), f"expected the document as an object, got {value!r}") -def _rendered(array: ArrayDocumentV3) -> dict[str, object]: +def _rendered(array: ArrayDocumentV3) -> dict[str, JSONValue]: """Each entity field the document has, as its entities write it; one nothing was read from is left out.""" - rendered: dict[str, object] = {} + rendered: dict[str, JSONValue] = {} for key, entity in ( ("data_type", array.data_type), ("chunk_grid", array.chunk_grid), @@ -294,24 +376,24 @@ def _as_written(original: object, rendered: object) -> object: return rendered -def _listed(document: Mapping[str, object], key: str) -> Sequence[object] | None: +def _listed(document: Mapping[str, JSONValue], key: str) -> Sequence[JSONValue] | None: """What the document lists at `key`; None if it wrote no array there.""" entries = document.get(key) - return cast("Sequence[object]", entries) if isinstance(entries, (list, tuple)) else None + return cast("Sequence[JSONValue]", entries) if isinstance(entries, (list, tuple)) else None def _read_one( - context: Context, kind: type[_EntityT], document: Mapping[str, object], key: str + context: Context, kind: type[_EntityT], document: Mapping[str, JSONValue], key: str ) -> tuple[_EntityT | Opaque, tuple[ValidationProblem, ...]]: """The entity of `kind` the document names at `key`; an `Opaque` if it names none.""" value = document.get(key) if value is None: - return Opaque(None, "invalid"), () - return resolve(value, kind, context, (key,), envelope_judged=True) + return Opaque.create_unchecked(None, "invalid"), () + return read_field(value, kind, context, (key,)) def _read_each( - context: Context, kind: type[_EntityT], document: Mapping[str, object], key: str + context: Context, kind: type[_EntityT], document: Mapping[str, JSONValue], key: str ) -> tuple[tuple[_EntityT | Opaque, ...], tuple[ValidationProblem, ...]]: """The entities of `kind` the document lists at `key`, in order.""" entries = _listed(document, key) @@ -320,42 +402,12 @@ def _read_each( read: list[_EntityT | Opaque] = [] problems: list[ValidationProblem] = [] for index, entry in enumerate(entries): - entity, found = resolve(entry, kind, context, (key, index), envelope_judged=True) + entity, found = read_field(entry, kind, context, (key, index)) read.append(entity) problems.extend(found) return tuple(read), tuple(problems) -def read_array_v3( - document: Mapping[str, object], context: Context -) -> tuple[ArrayDocumentV3, tuple[ValidationProblem, ...]]: - """`document`'s extension points, read in `context`. - - The one place that knows which of a document's fields holds which - kind of entity. Type-space only: what comes back is well-typed by - construction, and the problems are the reasons some of it is not an - entity. - """ - data_type, found_1 = _read_one(context, DataTypeEntity, document, "data_type") - chunk_grid, found_2 = _read_one(context, ChunkGridEntity, document, "chunk_grid") - encoding, found_3 = _read_one(context, ChunkKeyEncodingEntity, document, "chunk_key_encoding") - codecs, found_4 = _read_each(context, CodecEntity, document, "codecs") - transformers, found_5 = _read_each( - context, StorageTransformerEntity, document, "storage_transformers" - ) - return ( - ArrayDocumentV3( - document=document, - data_type=data_type, - chunk_grid=chunk_grid, - chunk_key_encoding=encoding, - codecs=codecs, - storage_transformers=transformers, - ), - (*found_1, *found_2, *found_3, *found_4, *found_5), - ) - - def _fill_value_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: """The fill value, judged by the data type it fills.""" if not isinstance(array.data_type, DataTypeEntity) or "fill_value" not in array.document: @@ -390,15 +442,16 @@ def _grid_problems(array: ArrayDocumentV3) -> tuple[ValidationProblem, ...]: def array_problems_v3( - document: Mapping[str, object], context: Context + document: Mapping[str, JSONValue], context: Context ) -> tuple[ValidationProblem, ...]: - """Every semantic problem in `document`, read in `context`. + """The second and third layers' problems for a refined document, together. - Expects a document the model layer has already accepted, so every - member is present and typed as its TypedDict declares. + For a document the first layer has refined; a consolidated child is + one. """ array, problems = read_array_v3(document, context) - return (*problems, *array.problems()) + _, composed = refine_array_v3(array) + return (*problems, *composed) def _prefixed(loc: Loc, problems: Sequence[ValidationProblem]) -> tuple[ValidationProblem, ...]: @@ -408,18 +461,18 @@ def _prefixed(loc: Loc, problems: Sequence[ValidationProblem]) -> tuple[Validati ) -def _as_string_mapping(value: object) -> Mapping[str, object] | None: +def _as_string_mapping(value: object) -> Mapping[str, JSONValue] | None: """`value` as a string-keyed mapping, or None if it is not one.""" if not isinstance(value, Mapping): return None mapping = cast("Mapping[object, object]", value) if any(not isinstance(key, str) for key in mapping): return None - return cast("Mapping[str, object]", mapping) + return cast("Mapping[str, JSONValue]", mapping) def group_problems_v3( - document: Mapping[str, object], context: Context = CORE_AND_EXTENSIONS + document: Mapping[str, JSONValue], context: Context = CORE_AND_EXTENSIONS ) -> tuple[ValidationProblem, ...]: """Every semantic problem in a v3 group document. @@ -469,8 +522,11 @@ def consolidated_entries_problems( __all__ = [ "ArrayDocumentV3", + "RefinedArrayV3", "array_problems_v3", "consolidated_entries_problems", "group_problems_v3", "read_array_v3", + "refine_array_v3", + "well_formed_array_v3", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index 9fc89422de..f9bdc58186 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -45,6 +45,8 @@ from zarr_metadata.model._validation import ( MetadataValidationError, ValidationProblem, + is_json, + refine_json, validate_metadata_field_v3, ) from zarr_metadata.v3._typed_json import ( @@ -53,7 +55,6 @@ Parser, RecordWriter, Writer, - as_tuples, declared_class_vars, field_hints, is_integer, @@ -205,21 +206,38 @@ class Opaque: `CodecEntity | Opaque` is written and simplified without asking which case it holds. - Built by the scope as it reads. The constructor refuses a `reason` - the reader does not give; `json` is whatever the document wrote, - which nothing checks, since the document may have written anything. + Built by the reader through `create_unchecked`, from JSON it has + refined; the constructor checks one built by hand. """ - json: object + json: JSONValue reason: Literal["out_of_scope", "invalid"] def __post_init__(self) -> None: - """Refuse a reason that is not one of the reader's two.""" + """Refuse a reason that is not one of the reader's two, and a `json` that is not JSON.""" reason: object = self.reason - if reason not in ("out_of_scope", "invalid"): - raise MetadataValidationError( - problem(("reason",), f"expected 'out_of_scope' or 'invalid', got {reason!r}") - ) + found = ( + *( + () + if reason in ("out_of_scope", "invalid") + else problem(("reason",), f"expected 'out_of_scope' or 'invalid', got {reason!r}") + ), + *( + () + if is_json(self.json) + else problem(("json",), f"expected JSON, got {self.json!r}") + ), + ) + if len(found) != 0: + raise MetadataValidationError(found) + + @classmethod + def create_unchecked(cls, json: JSONValue, reason: Literal["out_of_scope", "invalid"]) -> Self: + """An `Opaque` built without the constructor's check, for the reader, whose JSON is refined.""" + opaque = object.__new__(cls) + object.__setattr__(opaque, "json", json) + object.__setattr__(opaque, "reason", reason) + return opaque def to_json(self) -> ZarrV3MetadataFieldJSON: """The JSON the document wrote, as it wrote it. @@ -387,7 +405,9 @@ def parse(value: object, loc: Loc, reading: _Reading) -> Parsed: problems = is_metadata_field(value, loc) if len(problems) != 0: return value, problems - entity, found = resolve(value, kind, reading.context, loc) + # The parser hands refined JSON; its own type is `object` because + # the checker knows no JSON type. + entity, found = _resolve_field(cast("JSONValue", value), kind, reading.context, loc) reading.nested.extend(found) return entity, () @@ -413,55 +433,77 @@ def resolve( kind: type[EntityT], context: Context, loc: Loc = (), - *, - envelope_judged: bool = False, ) -> tuple[EntityT | Opaque, tuple[ValidationProblem, ...]]: """`data`, one metadata field, read as an entity of `kind` in `context`. - The reader. It relates the identifier in `data` to a concrete class - through `context`, and that class owns the validation routine: its - `coerce` is handed the field. What comes back is the entity, or an - `Opaque` saying why not. A name no class in `context` claims is - `out_of_scope` -- an unmodelled extension, left unjudged, which is - what makes the format open. A name claimed and refused, or of - another kind than this position takes, is `invalid`, for the reasons - reported alongside. `loc` prefixes the problems, so they point at - where in the containing configuration the field sat. - - A metadata field is a metadata field wherever it appears, so the - envelope gets the same structural judgment here that the model layer - gives a top-level one -- an extra member, a `configuration` that is - not an object, a `must_understand` that is not a boolean or is - `false`. `envelope_judged` says the model layer has judged and - reported that already, which it has for the fields of a document, so - it is neither judged nor reported twice. The class is asked whenever - there is one to ask -- a stray member or a malformed + The reader for one field: the first two layers of reading a + document, applied to a field on its own. The first needs nothing but + the value -- `data` is refined to JSON, arrays as tuples, and judged + as a metadata field, an extra member, a `configuration` that is not + an object or a `must_understand` that is not a boolean or is `false` + each a problem. The second needs `context`: the identifier in the + field is related to a concrete class through it, and that class owns + the validation routine, its `coerce`, which is handed the field. + + What comes back is the entity, or an `Opaque` saying why not. A name + no class in `context` claims is `out_of_scope` -- an unmodelled + extension, left unjudged, which is what makes the format open. A + name claimed and refused, or of another kind than this position + takes, is `invalid`, for the reasons reported alongside; so is a + value that is not JSON, or names no entity. `loc` prefixes the + problems, so they point at where in the containing configuration the + field sat. + """ + refined, problems = refine_json(data, loc) + if refined is None: + return Opaque.create_unchecked(None, "invalid"), problems + return _resolve_field(refined, kind, context, loc) + + +def _resolve_field( + data: JSONValue, kind: type[EntityT], context: Context, loc: Loc +) -> tuple[EntityT | Opaque, tuple[ValidationProblem, ...]]: + """A refined field, its envelope judged, then read. + + What a nested field gets: a metadata field is a metadata field + wherever it appears, so the envelope gets the same structural + judgment here that the model layer gives a top-level one. + """ + problems = tuple( + ValidationProblem((*loc, *found.loc), found.message, found.kind) + for found in validate_metadata_field_v3(data, allow_must_understand_false=False) + ) + entity, found = read_field(data, kind, context, loc) + return entity, (*problems, *found) + + +def read_field( + data: JSONValue, kind: type[EntityT], context: Context, loc: Loc = () +) -> tuple[EntityT | Opaque, tuple[ValidationProblem, ...]]: + """The second layer for one field whose envelope the first has judged. + + What `resolve` does once the envelope is judged, and what the + document reader does for a top-level field, whose envelope the model + layer judged with the document: relate the identifier in `data` to a + class through `context`, and hand the class the field. The class is + asked whenever there is one to ask -- a stray member or a malformed `must_understand` says nothing about the configuration -- and not when the value names no entity or its configuration is not an object, which the envelope judgment has said. """ - problems = ( - () - if envelope_judged - else tuple( - ValidationProblem((*loc, *found.loc), found.message, found.kind) - for found in validate_metadata_field_v3(data, allow_must_understand_false=False) - ) - ) name, _, malformed = named_configuration(data) if name is None or len(malformed) != 0: - return Opaque(data, "invalid"), problems + return Opaque.create_unchecked(data, "invalid"), () # Asked with the kind's kind, so a class of the wrong kind for this # position is found, and told apart from a name nothing claims. registered = kind_of(kind) entity_type = None if registered is None else context.claimant(registered, name) if entity_type is None: - return Opaque(data, "out_of_scope"), problems + return Opaque.create_unchecked(data, "out_of_scope"), () if not issubclass(entity_type, kind): # In scope, so not for another reader to resolve: the name is an # entity of the wrong kind for this position. - return Opaque(data, "invalid"), ( - *problems, + return Opaque.create_unchecked(data, "invalid"), ( ValidationProblem( loc, f"expected {_an(kind.__name__)}, got {name!r}, " @@ -470,12 +512,11 @@ def resolve( ), ) entity, found = entity_type.coerce(data, context) - problems = ( - *problems, - *(ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found), + problems = tuple( + ValidationProblem((*loc, *entry.loc), entry.message, entry.kind) for entry in found ) if entity is None: - return Opaque(data, "invalid"), problems + return Opaque.create_unchecked(data, "invalid"), problems return entity, problems @@ -793,11 +834,12 @@ def with_configuration(self, **changes: object) -> Self: return replace(self, configuration=replace(self.configuration, **changes)) @classmethod - def coerce(cls, value: object, context: Context) -> Coerced[Self]: + def coerce(cls, value: JSONValue, context: Context) -> Coerced[Self]: """`value` as this entity, or the reasons it is not one: the class's validation routine. `resolve` relates a field's name to this class and hands it the - field; this is what the class does with it. The configuration is + field, refined JSON with arrays as tuples, which is what `value` + is; this is what the class does with it. The configuration is parsed against the record the `configuration` field names, member by member; a member holding another entity is read in `context`, the scope this reading is happening in. An optional @@ -826,12 +868,8 @@ def coerce(cls, value: object, context: Context) -> Coerced[Self]: "missing_key", ) reading = _Reading(context, []) - # Arrays as tuples before parsing, so a member holds the tuples - # its type declares, never the lists raw JSON arrives as. A bare - # name's record has no members, so any key is an unknown one. - record, own = plan.read( - as_tuples({} if given is None else given), ("configuration",), reading - ) + # A bare name's record has no members, so any key is an unknown one. + record, own = plan.read({} if given is None else given, ("configuration",), reading) found = (*own, *reading.nested) if record is None: # An unknown key is survivable; a member that could not be @@ -929,6 +967,20 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb """ return () + def inner_pipelines( + self, incoming: ArrayParts | None + ) -> Mapping[str, tuple[Sequence[CodecEntity | Opaque], ArrayParts | None]]: + """The pipelines this codec holds, by the member holding each, with what each is handed. + + A shard holds two: its `codecs`, handed its inner chunk, and its + `index_codecs`, handed the shard index. Refinement walks them as + it walks the pipeline this codec stands in, locating what it + finds under the member, so a codec that holds pipelines says + which and what they receive, and judges nothing inside them + itself. Default: none. + """ + return {} + @dataclass(frozen=True) class ArrayArrayCodec(CodecEntity): @@ -1057,6 +1109,7 @@ def kind_of(cls: type[MetadataEntity]) -> type[MetadataEntity] | None: "named_configuration", "nested_kind", "problem", + "read_field", "resolve", "unreadable", "within", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py index d504fa2df3..95fb8668e9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_typed_json.py @@ -110,23 +110,6 @@ def is_integer(value: object) -> TypeIs[int]: return not isinstance(value, bool) and isinstance(value, int) -def as_tuples(value: object) -> object: - """Every JSON array in `value`, at any depth, as a tuple. - - The TypedDicts spell a JSON array as a tuple throughout, so a member - taken straight from parsed JSON would otherwise hold a list where its - own type says tuple -- and two documents differing only in that would - compare unequal. - """ - if isinstance(value, (list, tuple)): - entries = cast("list[object] | tuple[object, ...]", value) - return tuple(as_tuples(entry) for entry in entries) - if isinstance(value, Mapping): - entries = cast("Mapping[str, object]", value) - return {key: as_tuples(entry) for key, entry in entries.items()} - return value - - # --- annotations --------------------------------------------------------- @@ -865,7 +848,6 @@ def record_writer(record: type, leaf: WriterLeaf) -> RecordWriter: "Writer", "WriterLeaf", "any_of", - "as_tuples", "declared_class_vars", "describe", "each_of", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index 9ddc2f1a49..2a3c50726d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -11,7 +11,6 @@ from zarr_metadata.model._sentinel import UNSET from zarr_metadata.model._validation import ValidationProblem -from zarr_metadata.v3._chain import chain_problems from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( ArrayBytesCodec, @@ -29,7 +28,7 @@ from zarr_metadata.v3.data_type.uint64 import Uint64DataType if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Iterator, Mapping, Sequence SHARDING_INDEXED_CODEC_NAME: Final = "sharding_indexed" """The `name` field value of the `sharding_indexed` codec.""" @@ -140,50 +139,57 @@ def canonical(self) -> Self: ) def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: - """This shard against the array reaching it, and its two pipelines. + """This shard against the array reaching it. One sharding configuration encodes every chunk, so its inner shape has to divide all of them. Under a rectilinear grid an axis has several lengths and the inner extent must divide each; an axis whose lengths are unknown declines while the others are judged. + The index must be readable from metadata alone, so no codec of + variable output size may encode it. The two pipelines are + `inner_pipelines`, refined by the walk. + """ + return ( + *self._inner_chunk_problems(incoming), + *( + ValidationProblem( + ("index_codecs", index), + f"{type(codec).identifier!r} produces variable-size output; " + "index_codecs must be fixed-size", + "invalid_value", + ) + for index, codec in enumerate(self.configuration.index_codecs) + if isinstance(codec, CodecEntity) and type(codec).variable_size + ), + ) + + def inner_pipelines( + self, incoming: ArrayParts | None + ) -> "Mapping[str, tuple[Sequence[CodecEntity | Opaque], ArrayParts | None]]": + """The inner chunk pipeline and the index pipeline, with what each is handed. + + Both start from this codec's own configuration and from the + spec, so neither waits on what reached the codec. An unreadable + codec upstream costs the element type and the enclosing extents; + it does not make the inner chunk shape unknown, and the index is + a `uint64` array whatever precedes it. """ - found = list(self._inner_chunk_problems(incoming)) - # Both pipelines start from this codec's own configuration and - # from the spec, so neither waits on what reached the codec. An - # unreadable codec upstream costs the element type and the - # enclosing extents; it does not make the inner chunk shape - # unknown, and the index is a `uint64` array whatever precedes it. outer = incoming.grid if incoming is not None else UNKNOWN_GRID - found.extend( - chain_problems( + return { + "codecs": ( self.configuration.codecs, ArrayParts( ChunkGrid.regular(self.configuration.chunk_shape), incoming.data_type if incoming is not None else None, ), - ("codecs",), - ) - ) - found.extend( - chain_problems( + ), + "index_codecs": ( self.configuration.index_codecs, ArrayParts( shard_index_grid(outer, self.configuration.chunk_shape), Uint64DataType() ), - ("index_codecs",), - ) - ) - found.extend( - ValidationProblem( - ("index_codecs", index), - f"{type(codec).identifier!r} produces variable-size output; " - "index_codecs must be fixed-size", - "invalid_value", - ) - for index, codec in enumerate(self.configuration.index_codecs) - if isinstance(codec, CodecEntity) and type(codec).variable_size - ) - return tuple(found) + ), + } def _inner_chunk_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: """Whether the inner chunk divides every chunk this shard receives.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index 766bbd194a..dd56e594de 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -24,6 +24,19 @@ else: codec.json, codec.reason # 'out_of_scope': resolve it yourself +**Three layers**, ordered by what each needs, each handing the next a +typed value and its problems. `well_formed_array_v3(value)` needs only +the value: JSON refined, arrays as tuples, and the document's shape +judged. `read_array_v3(document, context)` needs a scope: each +extension point's name related to a class and the class handed the +field. `refine_array_v3(array)` needs the array: the fill value against +the type, the grid against the shape, and the codec pipeline walked with +what reaches each codec. Its value, `RefinedArrayV3`, is the resolved +pipeline -- `Pipeline` of `PipelineStage`, each a codec and the +`ArrayParts` it is handed, a shard's inner pipelines refined inside it +-- which is what a codec pipeline is built from; validation is what the +walk finds. `from_json` and `validate_array_metadata_v3` run all three. + **What comes back.** Problems, not exceptions, wherever a document is being judged rather than demanded. `zarr_metadata.rules.validate_array_metadata_v3(document, context=...)` returns a tuple of `ValidationProblem(loc, message, kind)`, each `loc` @@ -31,9 +44,10 @@ `("codecs", 1, "configuration", "level")`, and `kind` one of `invalid_type`, `invalid_value`, `missing_key`, `unknown_key` and `invalid_json`. `resolve(entry, CodecEntity, SCOPE)` reads one metadata -field as an entity of that kind: it relates the name in the entry to a -class in the scope and hands that class the field, since the class owns -its validation routine. It returns `(entity, problems)` where `entity` +field as an entity of that kind, the first two layers for a field on its +own: it refines and judges the field, relates the name in it to a class +in the scope, and hands that class the field, since the class owns its +validation routine. It returns `(entity, problems)` where `entity` is the entity or an `Opaque` -- never `None` -- with `loc` relative to the entry: `("configuration", "level")`. The class's routine, `coerce(value, context)`, returns `(entity or None, problems)`, that is @@ -160,7 +174,11 @@ class AcmeLz4Codec(BytesBytesCodec): -- one class for every `acme.fixedN` -- overrides `accepts(name)` and keeps the name in a field marked `Annotated[str, FROM_NAME]`, which `coerce` fills from the envelope. -- A codec: its kind is its base class. An `ArrayArrayCodec` defines +- A codec: its kind is its base class. One that holds pipelines of its + own, as a shard does, declares them through `inner_pipelines(incoming)` + -- each by the member that holds it, with the parts it is handed -- + and refinement walks them; it judges nothing inside them itself. An + `ArrayArrayCodec` defines `transition(incoming: ArrayParts) -> ArrayParts | None` -- abstract: return `incoming` if it leaves the array's shape, grid and data type alone, the parts it hands the next codec, or None when the metadata @@ -231,9 +249,15 @@ class AcmeLz4Codec(BytesBytesCodec): ProblemKind, ValidationProblem, ) -from zarr_metadata.v3._chain import chain_problems +from zarr_metadata.v3._chain import Pipeline, PipelineStage from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON -from zarr_metadata.v3._document import ArrayDocumentV3 +from zarr_metadata.v3._document import ( + ArrayDocumentV3, + RefinedArrayV3, + read_array_v3, + refine_array_v3, + well_formed_array_v3, +) from zarr_metadata.v3._entity import ( FROM_NAME, ArrayArrayCodec, @@ -293,15 +317,20 @@ class AcmeLz4Codec(BytesBytesCodec): "MetadataValidationError", "NumpyTimeDataType", "Opaque", + "Pipeline", + "PipelineStage", "ProblemKind", + "RefinedArrayV3", "StorageClass", "StorageTransformerEntity", "ValidationProblem", "ZarrV3MetadataFieldJSON", - "chain_problems", "is_integer", "named_configuration", "problem", + "read_array_v3", + "refine_array_v3", "resolve", + "well_formed_array_v3", "within", ] diff --git a/packages/zarr-metadata/tests/rules/test_chunk_grid.py b/packages/zarr-metadata/tests/rules/test_chunk_grid.py index b1d7d67ab8..a431d785eb 100644 --- a/packages/zarr-metadata/tests/rules/test_chunk_grid.py +++ b/packages/zarr-metadata/tests/rules/test_chunk_grid.py @@ -2,15 +2,18 @@ from __future__ import annotations -from collections.abc import Mapping +from typing import TYPE_CHECKING import pytest -from tests.helpers import configuration_of, entry_at +from tests.helpers import configuration_of from zarr_metadata.rules import validate_array_metadata_v3 from zarr_metadata.v3._parts import ChunkGrid, shard_index_grid from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS -from zarr_metadata.v3.entity import ChunkGridEntity +from zarr_metadata.v3.entity import ChunkGridEntity, resolve + +if TYPE_CHECKING: + from collections.abc import Mapping BASE: Mapping[str, object] = { "zarr_format": 3, @@ -93,20 +96,8 @@ def _grid_of(grid: object, shape: object) -> ChunkGrid: A grid entity builds its own; one out of scope pins only the rank the array shape gives it. """ - name = ( - grid - if isinstance(grid, str) - else entry_at(grid, "name") - if isinstance(grid, Mapping) - else None - ) - entity_type = ( - CORE_AND_EXTENSIONS.claimant(ChunkGridEntity, name) if isinstance(name, str) else None - ) - if entity_type is None: - return ChunkGrid.unreadable(shape) - entity, _ = entity_type.coerce(grid, CORE_AND_EXTENSIONS) - if entity is None: + entity, _ = resolve(grid, ChunkGridEntity, CORE_AND_EXTENSIONS) + if not isinstance(entity, ChunkGridEntity): return ChunkGrid.unreadable(shape) return entity.grid(shape) diff --git a/packages/zarr-metadata/tests/test_public_api.py b/packages/zarr-metadata/tests/test_public_api.py index e45e3e86d3..a5b7b2d9cf 100644 --- a/packages/zarr-metadata/tests/test_public_api.py +++ b/packages/zarr-metadata/tests/test_public_api.py @@ -303,6 +303,9 @@ def test_all_is_grouped_and_unique() -> None: "ChunkGrid", "ArrayParts", "ArrayDocumentV3", + "RefinedArrayV3", + "Pipeline", + "PipelineStage", "Endianness", "Invalid", "HexFloat16", diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 2b81ff939f..5f97c0d2cb 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -29,7 +29,7 @@ from tests.rules.strategies import valid_documents from zarr_metadata.model import UNSET, MetadataValidationError from zarr_metadata.rules import validate_array_metadata_v3 -from zarr_metadata.v3._document import read_array_v3 +from zarr_metadata.v3._document import read_array_v3, refine_array_v3, well_formed_array_v3 from zarr_metadata.v3._entity import BytesBytesCodec, Opaque, is_from_name from zarr_metadata.v3._registry import CORE, CORE_AND_EXTENSIONS from zarr_metadata.v3._typed_json import ( @@ -95,6 +95,7 @@ CodecEntity, Configuration, DataTypeEntity, + JSONValue, MetadataEntity, StorageTransformerEntity, resolve, @@ -255,7 +256,7 @@ } -def _round_trips(entity: type[MetadataEntity], document: object) -> MetadataEntity: +def _round_trips(entity: type[MetadataEntity], document: JSONValue) -> MetadataEntity: read, problems = entity.coerce(document, CORE_AND_EXTENSIONS) assert problems == () assert read is not None @@ -273,7 +274,7 @@ def _round_trips(entity: type[MetadataEntity], document: object) -> MetadataEnti ], ) def test_to_json_reads_back_to_the_same_entity( - entity: type[MetadataEntity], document: object + entity: type[MetadataEntity], document: JSONValue ) -> None: # What `to_json` writes, `coerce` reads to the entity that wrote it: # every member is written, in the spelling the reader expects. @@ -283,7 +284,10 @@ def test_to_json_reads_back_to_the_same_entity( @given(document=valid_documents()) @settings(max_examples=50, deadline=None) def test_to_json_reads_back_across_a_valid_document(document: dict[str, object]) -> None: - array, problems = read_array_v3(document, CORE_AND_EXTENSIONS) + refined, structural = well_formed_array_v3(document) + assert refined is not None + assert structural == () + array, problems = read_array_v3(refined, CORE_AND_EXTENSIONS) assert problems == () for entity in (array.data_type, array.chunk_grid, array.chunk_key_encoding, *array.codecs): assert isinstance(entity, MetadataEntity) @@ -517,7 +521,7 @@ def test_the_document_writes_back_only_the_fields_it_read() -> None: ], ids=["bare", "object", "empty-configuration", "must-understand", "both"], ) -def test_the_document_writes_an_envelope_as_it_was_written(spelling: object) -> None: +def test_the_document_writes_an_envelope_as_it_was_written(spelling: JSONValue) -> None: # An entity writes its own spelling; the document knows the one it # read and puts it back, around a codec in the pipeline and around # one inside a shard alike. Only `canonical` simplifies it. @@ -1098,3 +1102,108 @@ def test_create_unchecked_is_the_one_way_around_the_constructors() -> None: read, problems = GzipCodec.coerce({"name": "gzip", "configuration": {"level": 1}}, CORE) assert problems == () assert read == GzipCodec(GzipOptions(level=1)) + + +def test_the_first_layer_refines_json_and_judges_the_shape() -> None: + # Needs nothing but the value: arrays become tuples, the structural + # problems come with the document, and the next layer reads what it + # can regardless of them. + document, problems = well_formed_array_v3( + {"zarr_format": 3, "node_type": "array", "shape": [4]} + ) + assert document is not None + assert document["shape"] == (4,) + assert ("data_type",) in {problem.loc for problem in problems} + assert all(problem.kind == "missing_key" for problem in problems) + + +def test_error_a_value_that_is_not_json_gets_only_that_verdict() -> None: + document, problems = well_formed_array_v3({"shape": (4,), "attributes": {"x": object()}}) + assert document is None + assert [(p.loc, p.kind) for p in problems] == [(("attributes", "x"), "invalid_type")] + + +def test_error_a_document_that_is_not_an_object_has_nothing_to_read() -> None: + document, problems = well_formed_array_v3([1, 2]) + assert document is None + assert [(p.loc, p.kind) for p in problems] == [((), "invalid_type")] + + +def test_error_a_field_that_is_not_json_resolves_to_an_invalid_opaque() -> None: + entity, problems = resolve( + {"name": "gzip", "configuration": {"level": object()}}, CodecEntity, CORE + ) + assert entity == Opaque(None, "invalid") + assert [(p.loc, p.kind) for p in problems] == [(("configuration", "level"), "invalid_type")] + + +def test_the_third_layer_resolves_the_pipeline() -> None: + # What a codec pipeline is built from: at each position, the array + # that reaches the codec, and a shard's two pipelines refined inside + # it. Past the array->bytes boundary there is no array. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (64, 32), + "data_type": "float32", + "fill_value": 0.0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (32, 16)}}, + "chunk_key_encoding": "default", + "codecs": ( + {"name": "transpose", "configuration": {"order": (1, 0)}}, + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (8, 8), + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + "index_codecs": ( + {"name": "bytes", "configuration": {"endian": "little"}}, + "crc32c", + ), + }, + }, + {"name": "gzip", "configuration": {"level": 5}}, + ), + } + array = ArrayDocumentV3.from_json(document) + refined, problems = refine_array_v3(array) + assert problems == () + transpose, shard, gzip = refined.pipeline.stages + assert isinstance(transpose.codec, TransposeCodec) + assert transpose.incoming is refined.parts + assert refined.parts.grid.axis(0) == frozenset({32}) + assert shard.incoming is not None + assert shard.incoming.grid.axis(0) == frozenset({16}) # transposed + assert isinstance(shard.incoming.data_type, Float32DataType) + assert gzip.incoming is None + (inner_bytes,) = shard.inner["codecs"].stages + assert inner_bytes.incoming is not None + assert inner_bytes.incoming.grid.axis(1) == frozenset({8}) + assert isinstance(inner_bytes.incoming.data_type, Float32DataType) + index_bytes, index_crc = shard.inner["index_codecs"].stages + assert index_bytes.incoming is not None + assert isinstance(index_bytes.incoming.data_type, Uint64DataType) + assert index_crc.incoming is None + + +def test_an_out_of_scope_codec_receives_the_array_and_passes_nothing_on() -> None: + document = { + "shape": (4,), + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,)}}, + "codecs": ("acme.unknown", "bytes"), + } + array, problems = read_array_v3(well_formed_array_v3(document)[0] or {}, CORE_AND_EXTENSIONS) + assert problems == () + refined, _ = refine_array_v3(array) + unknown, bytes_codec = refined.pipeline.stages + assert isinstance(unknown.codec, Opaque) + assert unknown.incoming is refined.parts + assert bytes_codec.incoming is None + + +def test_error_an_opaque_holds_json() -> None: + with pytest.raises(MetadataValidationError) as caught: + Opaque({"name": object()}, "invalid") # pyright: ignore[reportArgumentType] + assert [(p.loc, p.kind) for p in caught.value.problems] == [(("json",), "invalid_type")] + assert Opaque.create_unchecked({"name": "x"}, "out_of_scope").reason == "out_of_scope" diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 985c0fdf42..420ba9b606 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -41,6 +41,7 @@ StorageClass, ValidationProblem, problem, + refine_array_v3, resolve, ) @@ -217,7 +218,7 @@ def test_a_reader_gets_entities_or_an_exception() -> None: array = ArrayDocumentV3.from_json(_document()) assert isinstance(array.data_type, DataTypeEntity) assert array.data_type.storage_class() == "single_byte" - assert array.parts.grid.rank == 1 + assert refine_array_v3(array)[0].parts.grid.rank == 1 assert [type(codec).identifier for codec in array.codecs if isinstance(codec, CodecEntity)] == [ "bytes" ] diff --git a/packages/zarr-metadata/tests/v3/test_fill_values.py b/packages/zarr-metadata/tests/v3/test_fill_values.py index 75e82c68cf..f0f97b5057 100644 --- a/packages/zarr-metadata/tests/v3/test_fill_values.py +++ b/packages/zarr-metadata/tests/v3/test_fill_values.py @@ -9,7 +9,6 @@ import pytest -from tests.helpers import entry_at from zarr_metadata.v3._registry import CORE_AND_EXTENSIONS from zarr_metadata.v3.entity import DataTypeEntity, resolve @@ -73,13 +72,9 @@ def _data_type(metadata: object) -> DataTypeEntity: - name = metadata if isinstance(metadata, str) else entry_at(metadata, "name") - assert isinstance(name, str), metadata - entity_type = CORE_AND_EXTENSIONS.claimant(DataTypeEntity, name) - assert entity_type is not None, metadata - entity, problems = entity_type.coerce(metadata, CORE_AND_EXTENSIONS) + entity, problems = resolve(metadata, DataTypeEntity, CORE_AND_EXTENSIONS) assert problems == (), problems - assert entity is not None, metadata + assert isinstance(entity, DataTypeEntity), metadata return entity From bc69fe6a2bcb269ff4d44a922ed60e57156eca9f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 20:53:53 +0200 Subject: [PATCH 103/107] fix(zarr-metadata): attributes are user data, and may hold a non-finite number zarr-python writes attributes with Python's `json` defaults, so a NaN `_FillValue` or CF `missing_value` is stored as a bare `NaN`. Every layer refused such a document, and the store reader could not decode it, so the consumer this layer is for could not read its own stores. `refine_node_json` is `refine_json` for a node document: a non-finite number is the float it is inside `attributes`, including the nodes a v3 group's inline `consolidated_metadata` holds, and not JSON anywhere else. The documents' first layer and the rules door read through it; `_validate_attributes` and the `is_*` guards agree with it. The store reader and writer judge what may be stored at a key by what the key stores (`stored_json_problems`): a `zarr.json` is a node document, a `.zattrs` is user data, a `.zmetadata` holds documents keyed the same way, a `.zarray` or `.zgroup` holds none. The reader now locates a non-finite number where it may not be instead of failing to decode, and the writer still refuses one, since a model built by hand is not validated. `is_json`, `validate_json` and `refine_json` stay RFC 8259. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems, 0 crashes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4379.bugfix.1.md | 15 ++ .../src/zarr_metadata/model/_array.py | 14 +- .../src/zarr_metadata/model/_group.py | 24 +- .../src/zarr_metadata/model/_validation.py | 215 ++++++++++++++---- .../src/zarr_metadata/rules/_documents.py | 4 +- .../src/zarr_metadata/v3/_document.py | 13 +- .../zarr-metadata/tests/model/test_array.py | 6 +- .../tests/model/test_refine_json.py | 163 +++++++++++++ .../tests/model/test_store_json.py | 151 ++++++++++++ .../tests/rules/test_documents.py | 13 ++ .../zarr-metadata/tests/v3/test_entities.py | 22 ++ 11 files changed, 581 insertions(+), 59 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.bugfix.1.md create mode 100644 packages/zarr-metadata/tests/model/test_refine_json.py create mode 100644 packages/zarr-metadata/tests/model/test_store_json.py diff --git a/packages/zarr-metadata/changes/4379.bugfix.1.md b/packages/zarr-metadata/changes/4379.bugfix.1.md new file mode 100644 index 0000000000..ecc0d6b9d7 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.bugfix.1.md @@ -0,0 +1,15 @@ +Attributes are user data, and a non-finite number in them is read, +validated and written back. zarr-python writes attributes with the +defaults of Python's `json` module, so an xarray `_FillValue` or a CF +`missing_value` of NaN is stored as a bare `NaN`, which RFC 8259 lacks. +This package refused such a document at every layer, and could not read +the store at all. A node's `attributes`, a v2 `.zattrs`, and the +attributes of the nodes an inline `consolidated_metadata` holds may now +hold `NaN`, `Infinity` and `-Infinity`, in every `validate_*`, +`parse_*` and `is_*`, and in `from_key_value` and `to_key_value`. + +Wherever the spec interprets a value it spells those numbers as strings, +so anywhere else a non-finite number is still refused. The store reader +now says where one is (`fill_value: non-finite float nan is not JSON`) +instead of failing to decode the document, and the writer still refuses +to write one, since a model built by hand is not validated. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 520c90ebbd..804c531b52 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -323,7 +323,11 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata: return cls.from_json(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)} + return { + ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json( + ZARR_V3_ARRAY_METADATA_STORE_KEY, self.to_json(), indent=indent + ) + } class ZarrV2ArrayMetadataPartial(TypedDict, total=False): @@ -489,8 +493,12 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes # when attributes are set (even empty) — UNSET emits no file. zarray = {k: v for k, v in self.to_json().items() if k != "attributes"} out: dict[str, bytes] = { - ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent) + ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json( + ZARR_V2_ARRAY_METADATA_STORE_KEY, zarray, indent=indent + ) } if self.attributes is not UNSET: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(self.attributes, indent=indent) + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( + ZARR_V2_ATTRIBUTES_STORE_KEY, self.attributes, indent=indent + ) return out diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 12e8324871..06d250bdc4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -24,8 +24,8 @@ load_store_json, parse_group_metadata_v2, parse_group_metadata_v3, + stored_json_problems, validate_consolidated_metadata_v3, - validate_json, ) from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY @@ -178,7 +178,11 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata: return cls.from_json(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY)) def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: - return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)} + return { + ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json( + ZARR_V3_GROUP_METADATA_STORE_KEY, self.to_json(), indent=indent + ) + } @dataclass(frozen=True, slots=True, kw_only=True) @@ -336,10 +340,14 @@ def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes # when attributes are set (even empty) — UNSET emits no file. zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"} out: dict[str, bytes] = { - ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent) + ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json( + ZARR_V2_GROUP_METADATA_STORE_KEY, zgroup, indent=indent + ) } if self.attributes is not UNSET: - out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(self.attributes, indent=indent) + out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json( + ZARR_V2_ATTRIBUTES_STORE_KEY, self.attributes, indent=indent + ) return out @@ -404,12 +412,14 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata: ) ) else: + # Each entry is the document its key names, so a `.zattrs` + # entry is user data (`stored_json_problems`). for key, value in cast("Mapping[str, object]", entries).items(): problems.extend( ValidationProblem( ("metadata", key, *problem.loc), problem.message, problem.kind ) - for problem in validate_json(value) + for problem in stored_json_problems(key, value) ) if len(problems) != 0: raise MetadataValidationError(problems) @@ -425,5 +435,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetad def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: return { - ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent) + ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json( + ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY, self.to_json(), indent=indent + ) } diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index b80ab6f6e7..15c6514b8c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -15,9 +15,9 @@ import json import math -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Final, Literal, NoReturn, cast +from typing import Final, Literal, cast from typing_extensions import TypeIs @@ -111,8 +111,16 @@ def _prefix( def validate_json(value: object) -> tuple[ValidationProblem, ...]: """Return every reason `value` is not JSON-serializable (recursively).""" + return _json_problems(value, finite=True) + + +def _json_problems(value: object, *, finite: bool) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not JSON, a non-finite number being one only when `finite`. + + `finite` is false for user data alone; see `refine_node_json`. + """ if isinstance(value, float): - if math.isfinite(value): + if not finite or math.isfinite(value): return () return (ValidationProblem((), f"non-finite float {value!r} is not JSON", "invalid_value"),) if isinstance(value, (str, int, bool)) or value is None: @@ -125,11 +133,11 @@ def validate_json(value: object) -> tuple[ValidationProblem, ...]: ValidationProblem((), f"non-string key {key!r} in JSON object", "invalid_type") ) continue - problems.extend(_prefix(key, validate_json(item))) + problems.extend(_prefix(key, _json_problems(item, finite=finite))) return tuple(problems) if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): for index, item in enumerate(cast("Sequence[object]", value)): - problems.extend(_prefix(index, validate_json(item))) + problems.extend(_prefix(index, _json_problems(item, finite=finite))) return tuple(problems) return (ValidationProblem((), f"not a JSON-serializable value: {value!r}", "invalid_type"),) @@ -146,8 +154,38 @@ def refine_json( JSON is None, with the problems located at the leaves that are not: not JSON is the first verdict, and there is nothing to read. """ + return _refine(value, loc, finite=True) + + +def refine_node_json( + value: object, loc: tuple[str | int, ...] = () +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: + """A node document refined as `refine_json` refines it, except for the user data it holds. + + A node's `attributes` are user data. The spec asks only that each be + a JSON value, no layer interprets one, and zarr-python writes them + with the defaults of Python's `json` module, so an attribute can hold + `NaN`, `Infinity` or `-Infinity` -- xarray's `_FillValue`, a CF + `missing_value` -- which RFC 8259 lacks. Such a number is read here + as the float it is. Wherever the spec interprets a value, it spells + those numbers as strings, so anywhere else a non-finite number is not + JSON, as in `refine_json`. The documents a v3 group's inline + `consolidated_metadata` holds are node documents too; a v2 document + is judged merged, its `.zattrs` as `attributes`. A value that is not + an object is refined as `refine_json` refines it. + """ + if not isinstance(value, Mapping): + return refine_json(value, loc) + return _refine_members(cast("Mapping[object, object]", value), loc, _node_member) + + +_Refined = tuple[JSONValue | None, tuple[ValidationProblem, ...]] + + +def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refined: + """`refine_json`, a non-finite number being JSON unless `finite`.""" if isinstance(value, float): - if math.isfinite(value): + if not finite or math.isfinite(value): return value, () return None, ( ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value"), @@ -155,24 +193,16 @@ def refine_json( if isinstance(value, (str, int, bool)) or value is None: return value, () if isinstance(value, Mapping): - members: dict[str, JSONValue] = {} - problems: list[ValidationProblem] = [] - for key, item in cast("Mapping[object, object]", value).items(): - if not isinstance(key, str): - problems.append( - ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") - ) - continue - member, found = refine_json(item, (*loc, key)) - problems.extend(found) - if len(found) == 0: - members[key] = member - return (members if len(problems) == 0 else None), tuple(problems) + return _refine_members( + cast("Mapping[object, object]", value), + loc, + lambda _key, item, at: _refine(item, at, finite=finite), + ) if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): entries: list[JSONValue] = [] found_in_entries: list[ValidationProblem] = [] for index, item in enumerate(cast("Sequence[object]", value)): - entry, found = refine_json(item, (*loc, index)) + entry, found = _refine(item, (*loc, index), finite=finite) found_in_entries.extend(found) if len(found) == 0: entries.append(entry) @@ -182,23 +212,74 @@ def refine_json( ) -def _is_canonical_json(value: object) -> TypeIs[JSONValue]: - """Whether `value` already uses the concrete containers in `JSONValue`.""" +def _refine_members( + value: Mapping[object, object], + loc: tuple[str | int, ...], + member: Callable[[str, object, tuple[str | int, ...]], _Refined], +) -> _Refined: + """An object refined member by member, in the order it was written, with `member`.""" + members: dict[str, JSONValue] = {} + problems: list[ValidationProblem] = [] + for key, item in value.items(): + if not isinstance(key, str): + problems.append( + ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") + ) + continue + refined, found = member(key, item, (*loc, key)) + problems.extend(found) + if len(found) == 0: + members[key] = refined + return (members if len(problems) == 0 else None), tuple(problems) + + +def _node_member(key: str, item: object, loc: tuple[str | int, ...]) -> _Refined: + """One member of a node document: its attributes are user data.""" + if key == "attributes": + return _refine(item, loc, finite=False) + if key == "consolidated_metadata" and isinstance(item, Mapping): + return _refine_members(cast("Mapping[object, object]", item), loc, _consolidated_member) + return _refine(item, loc, finite=True) + + +def _consolidated_member(key: str, item: object, loc: tuple[str | int, ...]) -> _Refined: + """One member of an inline consolidated envelope: its `metadata` maps paths to nodes.""" + if key == "metadata" and isinstance(item, Mapping): + return _refine_members( + cast("Mapping[object, object]", item), + loc, + lambda _path, node, at: refine_node_json(node, at), + ) + return _refine(item, loc, finite=True) + + +def _is_canonical_json(value: object, *, finite: bool = True) -> TypeIs[JSONValue]: + """Whether `value` already uses the concrete containers in `JSONValue`. + + A non-finite number counts only when `finite` is false; see + `_is_canonical_node_json`. + """ if isinstance(value, float): - return math.isfinite(value) + return not finite or math.isfinite(value) if isinstance(value, (str, int, bool)) or value is None: return True if isinstance(value, (list, tuple)): sequence = cast("list[object] | tuple[object, ...]", value) - return all(_is_canonical_json(item) for item in sequence) + return all(_is_canonical_json(item, finite=finite) for item in sequence) if isinstance(value, dict): mapping = cast("dict[object, object]", value) return all( - isinstance(key, str) and _is_canonical_json(item) for key, item in mapping.items() + isinstance(key, str) and _is_canonical_json(item, finite=finite) + for key, item in mapping.items() ) return False +def _is_canonical_node_json(value: object) -> bool: + """Whether a node document already uses `JSONValue`'s containers, and is JSON as `refine_node_json` reads one.""" + return _is_canonical_json(value, finite=False) and len(refine_node_json(value)[1]) == 0 + + def is_json(value: object) -> TypeIs[JSONValue]: """Whether `value` is a canonical JSON structure (recursively).""" return _is_canonical_json(value) @@ -559,9 +640,11 @@ def _validate_attributes(value: object) -> tuple[ValidationProblem, ...]: ("attributes",), "expected a mapping with string keys", "invalid_type" ), ) + # User data: a non-finite number is an attribute value (see + # `refine_node_json`). problems: list[ValidationProblem] = [] for key, item in cast("Mapping[str, object]", value).items(): - problems.extend(_prefix("attributes", _prefix(key, validate_json(item)))) + problems.extend(_prefix("attributes", _prefix(key, _json_problems(item, finite=False)))) return tuple(problems) @@ -651,7 +734,7 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: def is_array_metadata_v3(value: object) -> TypeIs[ZarrV3ArrayMetadataJSON]: """Whether `value` is a structurally-valid v3 array metadata document.""" return ( - _is_canonical_json(value) + _is_canonical_node_json(value) and not validate_array_metadata_v3(value) and _is_canonical_array_metadata_v3(value) ) @@ -743,7 +826,7 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: def is_array_metadata_v2(value: object) -> TypeIs[ZarrV2ArrayMetadataJSON]: """Whether `value` is a structurally-valid v2 array metadata document.""" return ( - _is_canonical_json(value) + _is_canonical_node_json(value) and not validate_array_metadata_v2(value) and _is_canonical_array_metadata_v2(value) ) @@ -855,7 +938,7 @@ def validate_group_metadata_v3(value: object) -> tuple[ValidationProblem, ...]: def is_group_metadata_v3(value: object) -> TypeIs[ZarrV3GroupMetadataJSON]: """Whether `value` is a structurally-valid v3 group metadata document.""" - return _is_canonical_json(value) and not validate_group_metadata_v3(value) + return _is_canonical_node_json(value) and not validate_group_metadata_v3(value) def parse_group_metadata_v3(value: object) -> ZarrV3GroupMetadataJSON: @@ -888,7 +971,7 @@ def validate_group_metadata_v2(value: object) -> tuple[ValidationProblem, ...]: def is_group_metadata_v2(value: object) -> TypeIs[ZarrV2GroupMetadataJSON]: """Whether `value` is a structurally-valid v2 group metadata document.""" - return _is_canonical_json(value) and not validate_group_metadata_v2(value) + return _is_canonical_node_json(value) and not validate_group_metadata_v2(value) def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: @@ -900,9 +983,47 @@ def parse_group_metadata_v2(value: object) -> ZarrV2GroupMetadataJSON: return cast(ZarrV2GroupMetadataJSON, normalized) -def _reject_json_constant(constant: str) -> NoReturn: - """Reject the JavaScript constants accepted by Python's JSON decoder.""" - raise ValueError(f"non-standard JSON constant {constant!r}") +def stored_json_problems(key: str, value: object) -> tuple[ValidationProblem, ...]: + """Every reason `value` is not JSON that may be stored at `key`. + + RFC 8259, except in user data, where a non-finite number is the value + it is (see `refine_node_json`). Which parts are user data follows + from what the key stores: a `zarr.json` is a node document, a + `.zattrs` is user data throughout, a `.zmetadata` holds documents + keyed the same way, and a `.zarray` or `.zgroup` holds none. + """ + name = key.rsplit("/", 1)[-1] + if name == "zarr.json": + return refine_node_json(value)[1] + if name == ".zattrs": + return _json_problems(value, finite=False) + if name == ".zmetadata" and isinstance(value, Mapping): + return _consolidated_v2_problems(cast("Mapping[object, object]", value)) + return validate_json(value) + + +def _consolidated_v2_problems( + consolidated: Mapping[object, object], +) -> tuple[ValidationProblem, ...]: + """A `.zmetadata` document: each entry judged as the document its key names.""" + entries = consolidated.get("metadata") + if not isinstance(entries, Mapping): + return validate_json(consolidated) + problems = list( + validate_json({key: item for key, item in consolidated.items() if key != "metadata"}) + ) + for entry_key, entry in cast("Mapping[object, object]", entries).items(): + if not isinstance(entry_key, str): + problems.append( + ValidationProblem( + ("metadata",), f"non-string key {entry_key!r} in JSON object", "invalid_type" + ) + ) + continue + problems.extend( + _prefix("metadata", _prefix(entry_key, stored_json_problems(entry_key, entry))) + ) + return tuple(problems) def load_store_json(mapping: Mapping[str, bytes], key: str) -> object: @@ -913,8 +1034,10 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> object: into typed positions silently. Narrow the result with a `parse_*`. Every ingestion failure surfaces as `MetadataValidationError`: a missing - store key is a `missing_key` problem and undecodable bytes are an - `invalid_json` problem, rather than leaking `KeyError` / + store key is a `missing_key` problem, undecodable bytes are an + `invalid_json` problem, and a non-finite number outside user data -- + which Python's decoder reads as a float -- is located where it was + written (`stored_json_problems`), rather than leaking `KeyError` / `json.JSONDecodeError` to callers. """ if key not in mapping: @@ -922,16 +1045,30 @@ def load_store_json(mapping: Mapping[str, bytes], key: str) -> object: [ValidationProblem((key,), "missing store key", "missing_key")] ) try: - return json.loads(mapping[key], parse_constant=_reject_json_constant) + value: object = json.loads(mapping[key]) except (UnicodeDecodeError, ValueError) as exc: raise MetadataValidationError( [ValidationProblem((key,), f"invalid JSON: {exc}", "invalid_json")] ) from exc + problems = stored_json_problems(key, value) + if len(problems) != 0: + raise MetadataValidationError(problems) + return value + +def dump_store_json(key: str, value: object, *, indent: int | str | None = None) -> bytes: + """Encode the document stored at `key` as JSON bytes. -def dump_store_json(value: object, *, indent: int | str | None = None) -> bytes: - """Encode a metadata document as strict RFC 8259 JSON bytes.""" - return json.dumps(value, indent=indent, allow_nan=False).encode("utf-8") + RFC 8259, except that a non-finite number in user data is written as + Python's `json` writes it (`NaN`, `Infinity`, `-Infinity`), which is + how zarr-python writes attributes. Anywhere else one is refused, as + `load_store_json` refuses it: a model built by hand is not validated, + so this is where a non-finite fill value would otherwise be written. + """ + problems = stored_json_problems(key, value) + if len(problems) != 0: + raise MetadataValidationError(problems) + return json.dumps(value, indent=indent, allow_nan=True).encode("utf-8") def arrays_to_tuples(obj: object) -> object: diff --git a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py index 2fc90caa56..68b9a7ff0f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py +++ b/packages/zarr-metadata/src/zarr_metadata/rules/_documents.py @@ -22,7 +22,7 @@ from zarr_metadata.model._array import ZarrV3ArrayMetadata from zarr_metadata.model._validation import ( MetadataValidationError, - refine_json, + refine_node_json, ) from zarr_metadata.model._validation import ( validate_array_metadata_v2 as _validate_structure_v2, @@ -106,7 +106,7 @@ def _judged( then the shape, then whatever the semantics need of an object. A value that is not JSON is None with only that verdict. """ - refined, problems = refine_json(value) + refined, problems = refine_node_json(value) if refined is None: return None, problems problems = structure(refined) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index 62692ce418..ed4817356f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -1,9 +1,9 @@ """A whole v3 array document, read in three layers, each with what it needs. 1. `well_formed_array_v3`: the value alone. JSON syntax and the - document's shape -- arrays as tuples, string keys, finite floats, the - keys a v3 array has and the shapes their values take, the envelope of - each extension point. + document's shape -- arrays as tuples, string keys, floats finite + outside the user's attributes, the keys a v3 array has and the shapes + their values take, the envelope of each extension point. 2. `read_array_v3`: a scope. Each extension point's name related to a class in the `Context`, and the class handed the field: the configuration parsed against its record, the rules asked, nested @@ -29,7 +29,7 @@ class and a registry entry, and this module does not change. from zarr_metadata.model._validation import ( MetadataValidationError, ValidationProblem, - refine_json, + refine_node_json, ) from zarr_metadata.model._validation import ( validate_array_metadata_v3 as validate_array_metadata_v3_structure, @@ -212,7 +212,8 @@ def well_formed_array_v3( """The first layer: `value` as a refined v3 array document, with every structural problem. Needs nothing but the value. The JSON is refined -- arrays as - tuples, string keys, finite floats -- and the document's shape is + tuples, string keys, floats finite except in the attributes, which + are user data (`refine_node_json`) -- and the document's shape is judged by the model layer: the keys a v3 array has, the shapes their values take, the envelope of each extension point. What comes back is refined JSON that the next layer reads without normalizing or @@ -221,7 +222,7 @@ def well_formed_array_v3( not JSON, or not an object, is None with the reasons: not JSON is the first verdict, and there is nothing to read. """ - refined, problems = refine_json(value) + refined, problems = refine_node_json(value) if refined is None: return None, problems if not isinstance(refined, Mapping): diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 3d4dbe05f6..9c84b5250c 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -1432,12 +1432,12 @@ def test_array_v3_from_json_materializes_abstract_containers() -> None: def test_from_key_value_rejects_non_standard_json_constant() -> None: - """Store JSON decoding rejects JavaScript NaN/Infinity constants.""" + """A JavaScript NaN constant outside the attributes is located, not decoded as a fill value.""" doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) doc["fill_value"] = float("nan") raw = json.dumps(doc) - with pytest.raises(MetadataValidationError, match="invalid JSON"): + with pytest.raises(MetadataValidationError, match="fill_value: non-finite float nan"): ZarrV3ArrayMetadata.from_key_value({"zarr.json": raw.encode()}) @@ -1445,7 +1445,7 @@ def test_to_key_value_rejects_non_finite_model_value() -> None: """Strict encoding prevents directly-constructed models from writing invalid JSON.""" model = ZarrV3ArrayMetadata.create_default(fill_value=float("nan")) - with pytest.raises(ValueError, match="JSON compliant"): + with pytest.raises(MetadataValidationError, match="fill_value: non-finite float nan"): model.to_key_value() diff --git a/packages/zarr-metadata/tests/model/test_refine_json.py b/packages/zarr-metadata/tests/model/test_refine_json.py new file mode 100644 index 0000000000..c87aaee7cc --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_refine_json.py @@ -0,0 +1,163 @@ +"""The first layer of reading: a value refined to JSON, or the reasons it is not.""" + +from __future__ import annotations + +import math +from collections import OrderedDict +from typing import TYPE_CHECKING, cast + +import pytest + +from zarr_metadata.model import MetadataValidationError, ValidationProblem +from zarr_metadata.model._validation import refine_json, refine_node_json, stored_json_problems + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + +@pytest.mark.parametrize( + ("value", "refined"), + [ + (1, 1), + (2.5, 2.5), + (True, True), + ("x", "x"), + (None, None), + ([1, [2, 3]], (1, (2, 3))), + ((1, 2), (1, 2)), + ({"a": [1], "b": {"c": None}}, {"a": (1,), "b": {"c": None}}), + (OrderedDict(k=[0]), {"k": (0,)}), + ], + ids=["int", "float", "bool", "str", "null", "nested-lists", "tuple", "object", "any-mapping"], +) +def test_json_is_refined_to_tuples_and_dicts(value: object, refined: object) -> None: + # One walk normalizes and judges; what comes back is what every later + # layer takes, and nothing later normalizes again. + assert refine_json(value) == (refined, ()) + + +@pytest.mark.parametrize( + ("value", "loc", "kind"), + [ + ({"a": object()}, ("a",), "invalid_type"), + ([1, [2, {3: 4}]], (1, 1), "invalid_type"), + ({"x": math.inf}, ("x",), "invalid_value"), + ({"x": [math.nan]}, ("x", 0), "invalid_value"), + (b"bytes", (), "invalid_type"), + ], + ids=["not-json-leaf", "non-string-key", "infinite", "nan-in-array", "bytes"], +) +def test_error_a_value_that_is_not_json_is_none_with_the_leaf_located( + value: object, loc: tuple[str | int, ...], kind: str +) -> None: + refined, problems = refine_json(value) + assert refined is None + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, kind)] + + +def test_error_every_leaf_that_is_not_json_is_reported() -> None: + refined, problems = refine_json({"a": object(), "b": [1, object()]}) + assert refined is None + assert [problem.loc for problem in problems] == [("a",), ("b", 1)] + + +def test_error_the_error_refuses_what_is_not_a_problem() -> None: + # `problem()` in the entity layer returns a one-element tuple; a list + # of those passes the type at the call and fails far away otherwise. + with pytest.raises(TypeError, match="collect with `extend`, not `append`"): + MetadataValidationError([(ValidationProblem(("a",), "bad a", "invalid_value"),)]) # pyright: ignore[reportArgumentType] + + +# A node's attributes are user data: Python's `json` writes a non-finite +# number there as `NaN`/`Infinity`, and zarr-python writes attributes that +# way, so the model reads, validates, and writes one there -- and nowhere +# else, where the spec spells those numbers as strings. + +_INLINE: dict[str, object] = {"kind": "inline", "must_understand": False} + + +def _at(value: object, path: tuple[str | int, ...]) -> object: + for part in path: + if isinstance(part, str): + value = cast("Mapping[str, object]", value)[part] + else: + value = cast("Sequence[object]", value)[part] + return value + + +@pytest.mark.parametrize( + ("document", "path"), + [ + ({"attributes": {"_FillValue": math.nan}}, ("attributes", "_FillValue")), + ({"attributes": {"valid_range": [-math.inf, math.inf]}}, ("attributes", "valid_range", 1)), + ( + {"attributes": {"cf": {"missing_value": -math.inf}}}, + ("attributes", "cf", "missing_value"), + ), + ( + { + "consolidated_metadata": { + **_INLINE, + "metadata": {"a": {"attributes": {"x": math.nan}}}, + } + }, + ("consolidated_metadata", "metadata", "a", "attributes", "x"), + ), + ], + ids=["nan", "infinities-in-an-array", "nested-object", "consolidated-node"], +) +def test_a_node_documents_attributes_hold_non_finite_numbers( + document: dict[str, object], path: tuple[str | int, ...] +) -> None: + refined, problems = refine_node_json(document) + assert problems == () + held = _at(refined, path) + assert isinstance(held, float) + assert not math.isfinite(held) + # What the model reads, it writes back, in the spelling Python's + # `json` module and zarr-python use. + assert stored_json_problems("zarr.json", document) == () + assert stored_json_problems(".zattrs", _at(document, path[:-1])) == () + + +@pytest.mark.parametrize( + ("document", "loc"), + [ + ({"fill_value": math.nan}, ("fill_value",)), + ( + {"codecs": ({"name": "scale_offset", "configuration": {"scale": math.inf}},)}, + ("codecs", 0, "configuration", "scale"), + ), + ({"extension": {"must_understand": False, "x": math.nan}}, ("extension", "x")), + ( + {"consolidated_metadata": {**_INLINE, "metadata": {"a": {"fill_value": math.nan}}}}, + ("consolidated_metadata", "metadata", "a", "fill_value"), + ), + ], + ids=["fill-value", "codec-configuration", "extension-field", "consolidated-node"], +) +def test_error_a_non_finite_number_outside_attributes_is_not_json( + document: dict[str, object], loc: tuple[str | int, ...] +) -> None: + refined, problems = refine_node_json(document) + assert refined is None + assert [(problem.loc, problem.kind) for problem in problems] == [(loc, "invalid_value")] + assert [problem.loc for problem in stored_json_problems("zarr.json", document)] == [loc] + + +def test_error_a_zarray_holds_no_user_data() -> None: + # A v2 array's attributes live in `.zattrs`; its `.zarray` is RFC 8259 + # throughout -- which is where zarr-python 3.0 once wrote a bare `NaN`. + problems = stored_json_problems("a/.zarray", {"fill_value": math.nan}) + assert [(problem.loc, problem.kind) for problem in problems] == [ + (("fill_value",), "invalid_value") + ] + + +def test_error_a_zmetadata_entry_is_judged_as_the_document_its_key_names() -> None: + consolidated = { + "zarr_consolidated_format": 1, + "metadata": {"a/.zattrs": {"x": math.nan}, "a/.zarray": {"fill_value": math.nan}}, + } + problems = stored_json_problems(".zmetadata", consolidated) + assert [problem.loc for problem in problems] == [("metadata", "a/.zarray", "fill_value")] diff --git a/packages/zarr-metadata/tests/model/test_store_json.py b/packages/zarr-metadata/tests/model/test_store_json.py new file mode 100644 index 0000000000..34ac2ca63a --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_store_json.py @@ -0,0 +1,151 @@ +"""What the model reads from and writes to a store, key by key. + +A node's attributes are user data, and zarr-python writes them with the +defaults of Python's `json` module, so an attribute may hold `NaN`, +`Infinity` or `-Infinity`. The model reads such a store, validates it, and +writes it back the same way; anywhere else a non-finite number is refused. +""" + +from __future__ import annotations + +import json +import math +from typing import TYPE_CHECKING, Any, Protocol, cast + +import pytest + +from zarr_metadata.model import ( + MetadataValidationError, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3GroupMetadata, + is_array_metadata_v2, + is_array_metadata_v3, + is_group_metadata_v2, + is_group_metadata_v3, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + +class _Stored(Protocol): + """A model that writes itself to a store.""" + + def to_json(self) -> Mapping[str, object]: ... + + def to_key_value(self, *, indent: int | str | None = None) -> Mapping[str, bytes]: ... + + +ATTRIBUTES: dict[str, object] = {"_FillValue": math.nan, "valid_range": [-math.inf, math.inf]} + + +def _held(attributes: object) -> tuple[bool, bool, bool]: + held = cast("Mapping[str, object]", attributes) + fill = held["_FillValue"] + low, high = cast("tuple[float, float]", held["valid_range"]) + return (isinstance(fill, float) and math.isnan(fill), low == -math.inf, high == math.inf) + + +def _v3_array() -> Mapping[str, bytes]: + document = {**ZarrV3ArrayMetadata.create_default().to_json(), "attributes": ATTRIBUTES} + return {"zarr.json": json.dumps(document).encode()} + + +def _v3_group() -> Mapping[str, bytes]: + document = {"zarr_format": 3, "node_type": "group", "attributes": ATTRIBUTES} + return {"zarr.json": json.dumps(document).encode()} + + +def _v2_array() -> Mapping[str, bytes]: + zarray = dict(ZarrV2ArrayMetadata.create_default().to_json()) + return {".zarray": json.dumps(zarray).encode(), ".zattrs": json.dumps(ATTRIBUTES).encode()} + + +def _v2_group() -> Mapping[str, bytes]: + return { + ".zgroup": json.dumps({"zarr_format": 2}).encode(), + ".zattrs": json.dumps(ATTRIBUTES).encode(), + } + + +def _v2_consolidated() -> Mapping[str, bytes]: + document = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}, ".zattrs": ATTRIBUTES}, + } + return {".zmetadata": json.dumps(document).encode()} + + +@pytest.mark.parametrize( + ("store", "read", "attributes_of", "guard"), + [ + ( + _v3_array, + ZarrV3ArrayMetadata.from_key_value, + lambda model: model.attributes, + is_array_metadata_v3, + ), + ( + _v3_group, + ZarrV3GroupMetadata.from_key_value, + lambda model: model.attributes, + is_group_metadata_v3, + ), + ( + _v2_array, + ZarrV2ArrayMetadata.from_key_value, + lambda model: model.attributes, + is_array_metadata_v2, + ), + ( + _v2_group, + ZarrV2GroupMetadata.from_key_value, + lambda model: model.attributes, + is_group_metadata_v2, + ), + ( + _v2_consolidated, + ZarrV2ConsolidatedMetadata.from_key_value, + lambda model: model.metadata[".zattrs"], + None, + ), + ], + ids=["v3-array", "v3-group", "v2-array", "v2-group", "v2-consolidated"], +) +def test_attributes_holding_non_finite_numbers_round_trip_through_the_store( + store: Callable[[], Mapping[str, bytes]], + read: Callable[[Mapping[str, bytes]], _Stored], + attributes_of: Callable[[Any], object], + guard: Callable[[object], bool] | None, +) -> None: + model = read(store()) + assert _held(attributes_of(model)) == (True, True, True) + if guard is not None: + assert guard(model.to_json()) + written = model.to_key_value() + assert b"NaN" in b"".join(written.values()) + assert _held(attributes_of(read(written))) == (True, True, True) + + +def test_error_a_non_finite_number_outside_attributes_is_located_when_read() -> None: + # Python's decoder reads a bare `NaN` as a float; where it may not be, + # the store says where it is. + zarray = dict(ZarrV2ArrayMetadata.create_default().to_json()) + zarray["fill_value"] = math.nan + with pytest.raises(MetadataValidationError) as raised: + ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(zarray).encode()}) + assert [(problem.loc, problem.kind) for problem in raised.value.problems] == [ + (("fill_value",), "invalid_value") + ] + + +def test_error_a_non_finite_number_outside_attributes_is_not_written() -> None: + # A model built by hand is not validated; the writer is what stands + # between a non-finite fill value and a document no reader accepts. + model = ZarrV3ArrayMetadata.create_default(fill_value=math.nan, attributes={"x": math.nan}) + with pytest.raises(MetadataValidationError) as raised: + model.to_key_value() + assert [problem.loc for problem in raised.value.problems] == [("fill_value",)] diff --git a/packages/zarr-metadata/tests/rules/test_documents.py b/packages/zarr-metadata/tests/rules/test_documents.py index 0e34ef1a2b..1ef75df6dc 100644 --- a/packages/zarr-metadata/tests/rules/test_documents.py +++ b/packages/zarr-metadata/tests/rules/test_documents.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from typing import TYPE_CHECKING import pytest @@ -70,6 +71,18 @@ parse_array_metadata_v2, V2_ARRAY, ), + # Attributes are user data: zarr-python writes a non-finite number + # there as Python's `json` does, and every layer reads it. + "v3-array-attributes-hold-non-finite-numbers": ( + validate_array_metadata_v3, + parse_array_metadata_v3, + {**V3_ARRAY, "attributes": {"_FillValue": math.nan, "range": [-math.inf, math.inf]}}, + ), + "v2-array-attributes-hold-non-finite-numbers": ( + validate_array_metadata_v2, + parse_array_metadata_v2, + {**V2_ARRAY, "attributes": {"_FillValue": math.nan}}, + ), } diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 5f97c0d2cb..7006cb86b7 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -11,6 +11,7 @@ import copy import dataclasses +import math import sys from typing import ( Any, @@ -1207,3 +1208,24 @@ def test_error_an_opaque_holds_json() -> None: Opaque({"name": object()}, "invalid") # pyright: ignore[reportArgumentType] assert [(p.loc, p.kind) for p in caught.value.problems] == [(("json",), "invalid_type")] assert Opaque.create_unchecked({"name": "x"}, "out_of_scope").reason == "out_of_scope" + + +def test_the_document_reads_and_writes_attributes_as_user_data() -> None: + # zarr-python writes attributes with Python's `json` defaults, so an + # xarray `_FillValue` of NaN arrives as a bare `NaN`. Attributes are + # nobody's to interpret: every layer reads one, and the faithful + # writer puts it back. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "float32", + "fill_value": "NaN", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,)}}, + "chunk_key_encoding": "default", + "codecs": ({"name": "bytes", "configuration": {"endian": "little"}},), + "attributes": {"_FillValue": math.nan}, + } + written = ArrayDocumentV3.from_json(document).to_json()["attributes"] + fill = cast("dict[str, float]", written)["_FillValue"] + assert math.isnan(fill) From afb75510a9d1a7380db283068098957894166f8d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 20:57:27 +0200 Subject: [PATCH 104/107] feat(zarr-metadata): the read document names the fields a reader must understand The spec has a reader refuse an array carrying a top-level field it does not recognize unless the field says `must_understand: false`. The model partitions by that obligation (`must_understand_fields`) and leaves recognition to the reader; the read document offered no way to ask, so a consumer had to walk the raw document. It now has the same property. A verdict at the second layer was the alternative. It would close the top-level namespace the model keeps open (pinned by `test_v3_array_schema_allows_unknown_extension_fields`), and treat an unknown field unlike an unknown codec, which comes back `Opaque` for its consumer to judge. The door's reading example called `array.parts`, which the third layer owns now; it asks this instead. Unchanged to the 40k-document corpus: no verdict changes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4379.feature.12.md | 7 ++++++ .../src/zarr_metadata/v3/_document.py | 22 +++++++++++++++++++ .../src/zarr_metadata/v3/entity.py | 6 +++-- .../zarr-metadata/tests/v3/test_entities.py | 22 +++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 packages/zarr-metadata/changes/4379.feature.12.md diff --git a/packages/zarr-metadata/changes/4379.feature.12.md b/packages/zarr-metadata/changes/4379.feature.12.md new file mode 100644 index 0000000000..3c9094391e --- /dev/null +++ b/packages/zarr-metadata/changes/4379.feature.12.md @@ -0,0 +1,7 @@ +`ArrayDocumentV3.must_understand_fields` names the top-level fields +outside the spec's that do not say `must_understand: false`, as +`ZarrV3ArrayMetadata.must_understand_fields` does for the model. The spec +has a reader refuse to open an array carrying such a field unless it +recognizes it, and recognition is the reader's own knowledge, so the +document keeps the field, writes it back, and leaves the refusal to its +reader: zarr-python recognizes none. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py index ed4817356f..bfebfdf6c7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_document.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_document.py @@ -26,7 +26,9 @@ class and a registry entry, and this module does not change. from dataclasses import dataclass, replace from typing import TYPE_CHECKING, TypeVar, cast +from zarr_metadata.model._array import must_understand_subset from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, MetadataValidationError, ValidationProblem, refine_node_json, @@ -56,6 +58,7 @@ class and a registry entry, and this module does not change. from zarr_metadata._common import JSONValue from zarr_metadata.v3._entity import Loc + from zarr_metadata.v3.array import ZarrV3ExtensionField _EntityT = TypeVar("_EntityT", bound=MetadataEntity) @@ -160,6 +163,25 @@ def to_json(self) -> dict[str, JSONValue]: }, } + @property + def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: + """The fields outside the spec's that do not say `must_understand: false`. + + A reader must refuse to open the array if this holds a field it + does not recognize. Recognition is the reader's own knowledge: a + top-level field is no extension point, so no scope claims one, + and the document partitions by obligation and leaves the verdict + to its reader, as `ZarrV3ArrayMetadata.must_understand_fields` + does. An extension point the scope does not claim is the same + question asked of an `Opaque`. + """ + extra = { + key: value + for key, value in self.document.items() + if key not in ARRAY_METADATA_STANDARD_KEYS_V3 + } + return must_understand_subset(cast("Mapping[str, ZarrV3ExtensionField]", extra)) + @classmethod def from_json(cls, value: object, *, context: Context = CORE_AND_EXTENSIONS) -> ArrayDocumentV3: """A v3 array document read into entities, or raise. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index dd56e594de..fe0978b2ed 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -12,12 +12,14 @@ spell out -- what a data type's scalars are, which position a codec occupies, what a grid divides an array into. A name the scope does not model is not a failure: it arrives as an `Opaque` marked `out_of_scope`, -for the reader to resolve elsewhere. +for the reader to resolve elsewhere. Nor is a top-level field outside the +spec's: `must_understand_fields` names the ones the reader must refuse +to open the array without recognizing. from zarr_metadata.v3.entity import ArrayBytesCodec, ArrayDocumentV3, CodecEntity array = ArrayDocumentV3.from_json(json.loads(raw)) # or raises - array.parts.grid.rank + array.must_understand_fields # a field here you do not know: refuse for codec in array.codecs: if isinstance(codec, CodecEntity): isinstance(codec, ArrayBytesCodec) # its pipeline position is its base class diff --git a/packages/zarr-metadata/tests/v3/test_entities.py b/packages/zarr-metadata/tests/v3/test_entities.py index 7006cb86b7..1c6d0cd5fa 100644 --- a/packages/zarr-metadata/tests/v3/test_entities.py +++ b/packages/zarr-metadata/tests/v3/test_entities.py @@ -1229,3 +1229,25 @@ def test_the_document_reads_and_writes_attributes_as_user_data() -> None: written = ArrayDocumentV3.from_json(document).to_json()["attributes"] fill = cast("dict[str, float]", written)["_FillValue"] assert math.isnan(fill) + + +def test_the_document_names_the_fields_a_reader_must_understand() -> None: + # Recognition is the reader's own knowledge, as in the model: the + # document keeps an unknown field, writes it back, and says which + # ones a reader must refuse to open the array without understanding. + document = { + "zarr_format": 3, + "node_type": "array", + "shape": (4,), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2,)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), + "provenance": {"tool": "x"}, + "notes": {"must_understand": False, "text": "y"}, + "flag": True, + } + array = ArrayDocumentV3.from_json(document) + assert array.must_understand_fields == {"provenance": {"tool": "x"}, "flag": True} + assert array.to_json() == document From 6e5c3c7e96fd5f15f6a600cb6111869c19848bdb Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 20:59:47 +0200 Subject: [PATCH 105/107] fix(zarr-metadata): scale_offset's scalars are fill values of the type it receives The registry says `offset` and `scale` are each "encoded to JSON using the Zarr V3 fill value encoding for the input array's data type". The entity took any JSON value and nothing asked the data type, so the string "0" passed as a float32 scale -- the shape of the zarr-python bug where a string zero slipped past its zero check. `incoming_problems` asks the type that reaches the codec, which after a `cast_value` is the type cast to, to judge each scalar as a fill value, and refuses the codec on a type without arithmetic: the registry lists the integer and floating-point types, so the allow-list is the two families. A zero `scale` is not refused: the spec does not forbid one, and only a reader that decodes every spelling of zero can say which scalars are zero. Against the 40k-document corpus: 19 documents valid before are invalid, each for a scalar no data type could hold; 45 problems gained, none lost. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4379.bugfix.2.md | 10 ++++ .../zarr_metadata/v3/codec/scale_offset.py | 36 ++++++++++++- .../tests/rules/test_v3_array_rules.py | 53 +++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/changes/4379.bugfix.2.md diff --git a/packages/zarr-metadata/changes/4379.bugfix.2.md b/packages/zarr-metadata/changes/4379.bugfix.2.md new file mode 100644 index 0000000000..c7d0343fb3 --- /dev/null +++ b/packages/zarr-metadata/changes/4379.bugfix.2.md @@ -0,0 +1,10 @@ +The `scale_offset` codec's `offset` and `scale` are judged. The registry +says each is "encoded to JSON using the Zarr V3 fill value encoding for +the input array's data type", so each is now a fill value of the data type +that reaches the codec, which after a `cast_value` is the type cast to: +the string `"0"` is neither a `float32` nor an `int32`, and `-1` is no +`uint8`. Any JSON value was accepted before. The codec is defined for data +types with arithmetic, and the registry lists the integer and +floating-point ones, so on a `bool` or `complex64` array it is refused at +the codec. Over the 40,000-document corpus, 19 documents that were valid +are not, each for a scalar no data type could hold (an object as `scale`). diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 6e62c4e31c..9697aeb7cb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -15,8 +15,11 @@ from zarr_metadata.v3._entity import ( ArrayArrayCodec, Configuration, + DataTypeEntity, + problem, ) from zarr_metadata.v3._parts import ArrayParts +from zarr_metadata.v3.data_type._families import FloatDataType, IntegerDataType if TYPE_CHECKING: from collections.abc import Iterator @@ -103,7 +106,7 @@ class ScaleOffsetCodec(ArrayArrayCodec): Both members are optional and any JSON scalar is well-typed here; what a given value means depends on the data type it is applied to, which - is a question for the rules layer. + `incoming_problems` asks of the type that reaches the codec. """ configuration: ScaleOffsetOptions @@ -111,6 +114,37 @@ class ScaleOffsetCodec(ArrayArrayCodec): identifier: ClassVar[str] = SCALE_OFFSET_CODEC_NAME variable_size: ClassVar[bool] = False + def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProblem, ...]: + """What the array handed to this codec must be, and what its members must be for it. + + The registry defines the codec for data types with arithmetic and + lists the integer and floating-point ones. `offset` and `scale` + are each "encoded to JSON using the Zarr V3 fill value encoding + for the input array's data type" -- the type that reaches this + codec, which after a `cast_value` is not the array's own -- so + each is a fill value of that type, and that type judges it: the + string `"0"` is no `float32` and no `int32`. + """ + data_type = incoming.data_type if incoming is not None else None + if not isinstance(data_type, DataTypeEntity): + return () + if not isinstance(data_type, (IntegerDataType, FloatDataType)): + return problem( + (), + "scale_offset is defined for integer and floating-point data types, not " + f"{type(data_type).identifier!r}", + "invalid_value", + ) + return tuple( + found + for member, value in ( + ("offset", self.configuration.offset), + ("scale", self.configuration.scale), + ) + if value is not UNSET + for found in data_type.fill_value_problems(value, (member,)) + ) + def transition(self, incoming: ArrayParts) -> ArrayParts | None: """The same array, element for element. diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index e4b4530796..9495f9b6aa 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -116,6 +116,20 @@ def _shard(**overrides: object) -> Mapping[str, object]: "chunk_grid": {"name": "hilbert", "configuration": {"level": 3}}, }, "unknown-codec-inconclusive": {**BASE, "codecs": ({"name": "zfpy"}, "bytes")}, + "scale-offset-scalars-of-the-array-type": { + **BASE, + "codecs": ({"name": "scale_offset", "configuration": {"offset": 1, "scale": 2}}, "bytes"), + }, + "scale-offset-scalars-of-the-type-cast-to": { + **BASE, + "data_type": "float32", + "fill_value": "NaN", + "codecs": ( + {"name": "cast_value", "configuration": {"data_type": "uint8"}}, + {"name": "scale_offset", "configuration": {"offset": 10}}, + "bytes", + ), + }, } @@ -801,3 +815,42 @@ def test_error_a_pipeline_that_is_not_an_array_is_not_judged_as_empty() -> None: # codec missing from: the one problem is the shape of the field. problems = validate_array_metadata_v3(cast("Any", {**BASE, "codecs": "bytes"})) assert [(problem.loc, problem.kind) for problem in problems] == [(("codecs",), "invalid_type")] + + +@pytest.mark.parametrize( + ("codecs", "loc"), + [ + ( + ({"name": "scale_offset", "configuration": {"scale": "0"}}, "bytes"), + ("codecs", 0, "configuration", "scale"), + ), + ( + ({"name": "scale_offset", "configuration": {"offset": -1}}, "bytes"), + ("codecs", 0, "configuration", "offset"), + ), + ( + ( + {"name": "cast_value", "configuration": {"data_type": "uint8"}}, + {"name": "scale_offset", "configuration": {"scale": 0.5}}, + "bytes", + ), + ("codecs", 1, "configuration", "scale"), + ), + ], + ids=["string-for-uint8", "out-of-range-for-uint8", "float-for-the-type-cast-to"], +) +def test_error_scale_offset_scalar_is_no_fill_value_of_the_type_it_receives( + codecs: tuple[object, ...], loc: tuple[str | int, ...] +) -> None: + # Each scalar is "encoded to JSON using the Zarr V3 fill value encoding + # for the input array's data type" -- the type that reaches the codec. + problems = validate_array_metadata_v3({**BASE, "codecs": codecs}) + assert [(p.loc, p.kind) for p in problems] == [(loc, "invalid_value")] + + +def test_error_scale_offset_data_type_has_no_arithmetic() -> None: + doc = {**BASE, "data_type": "bool", "fill_value": False, "codecs": ("scale_offset", "bytes")} + assert _sole_problem(doc) == ( + ("codecs", 0), + "scale_offset is defined for integer and floating-point data types, not 'bool'", + ) From cea8fbd5e987a9c39f0ab0ad4639a754efc91863 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 21:03:24 +0200 Subject: [PATCH 106/107] docs(zarr-metadata): the differential numbers, re-derived at this head The fragment quoted 4,397 fewer and 15,259 more problems among documents invalid under both trees, counted at an earlier head; no counting reproduces them at 9c844e266, where the duplicate-envelope and phantom-pipeline reports were already gone. Re-derived per entry point against c0600eef8 at this head: 0 invalid->valid, 21 valid->invalid (2 `must_understand: false`, 19 `scale_offset` scalars), 6,382 fewer, 655 more. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4379.feature.10.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/zarr-metadata/changes/4379.feature.10.md b/packages/zarr-metadata/changes/4379.feature.10.md index 132447b434..a25f5fdc87 100644 --- a/packages/zarr-metadata/changes/4379.feature.10.md +++ b/packages/zarr-metadata/changes/4379.feature.10.md @@ -31,12 +31,16 @@ standing in for the entity. Measured against the rule registry this layer replaced, over a shared corpus of 40,000 documents through all four `validate_*` entry points: -**nothing this layer accepts was rejected before**. Two verdicts change, -both the other way and both the `must_understand: false` refusal above -- -one at the top level, one nested in a shard's pipelines. Every document -valid under both reports identically. Among those invalid under both, -4,397 report fewer problems and 15,259 report more, the latter because -a problem that used to stand down the rest of an entity no longer does. +**nothing this layer accepts was rejected before**. Twenty-one verdicts +change, all the other way: two are the `must_understand: false` refusal +above -- one at the top level, one nested in a shard's pipelines -- and +nineteen are `scale_offset` scalars no data type could hold. Every +document valid under both reports identically. Counted per entry point, +among those invalid under both, 6,382 report fewer problems -- a +malformed envelope is reported once, and a document with no codec list +is no longer also told it lacks an `array -> bytes` codec -- and 655 +report more, because a problem that used to stand down the rest of an +entity no longer does. Some ways of writing an entity type-check cleanly and then fail somewhere that will not name the class, so registration refuses them: a From eab9ec74aca936165fd0dc2e3aeff66a9ce56b94 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 22 Sep 2026 21:06:35 +0200 Subject: [PATCH 107/107] feat(zarr-metadata): reject cast_value wrap on non-integer targets `out_of_range: "wrap"` is defined only for integral targets with a two's complement representation. A modelled non-integer target -- `bool` included, which is not two's complement -- is rejected; a target out of scope declines, since it may be an integral extension type. Re-landed on the current entity layer. The rule is the `cast_value` record's `problems`, and whether a type wraps is a fact each data type states, `twos_complement`, owed by `DataTypeEntity`: registration refuses a data type that has not decided, so there is no table of names for a test to keep in step. Entities also gain `name`, the spelling a document writes, distinct from `identifier`, the key they are tabled under. They differ only for the raw-bytes family, whose identifier is invented; the `bytes` and `scale_offset` messages now name an `r24` as `r24` too. Unchanged to the 40k-document corpus: 0 verdicts, 0 problems, 0 crashes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4380.feature.md | 17 +++++++ .../src/zarr_metadata/v3/_entity.py | 23 +++++++++ .../src/zarr_metadata/v3/codec/bytes.py | 2 +- .../src/zarr_metadata/v3/codec/cast_value.py | 27 +++++++++- .../zarr_metadata/v3/codec/scale_offset.py | 2 +- .../zarr_metadata/v3/data_type/_families.py | 6 +++ .../src/zarr_metadata/v3/data_type/bool.py | 1 + .../src/zarr_metadata/v3/data_type/bytes.py | 1 + .../src/zarr_metadata/v3/data_type/raw.py | 1 + .../src/zarr_metadata/v3/data_type/string.py | 1 + .../src/zarr_metadata/v3/data_type/struct.py | 1 + .../src/zarr_metadata/v3/entity.py | 9 ++-- .../tests/rules/test_v3_array_rules.py | 49 +++++++++++++++++++ .../tests/v3/test_acme_decimal.py | 1 + .../tests/v3/test_extension_api.py | 21 ++++++++ 15 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 packages/zarr-metadata/changes/4380.feature.md diff --git a/packages/zarr-metadata/changes/4380.feature.md b/packages/zarr-metadata/changes/4380.feature.md new file mode 100644 index 0000000000..0ffec7cc4b --- /dev/null +++ b/packages/zarr-metadata/changes/4380.feature.md @@ -0,0 +1,17 @@ +`cast_value`'s `out_of_range: "wrap"` is rejected for targets it is not +defined on. The spec permits wrapping only for integral targets with a +two's complement representation, so a known non-integer target -- `bool` +included, which is not two's complement -- reports `invalid_value` at +`out_of_range`. A target this reader does not model declines, since it +may be an integral extension type. + +Whether a data type wraps is a fact the data type states, +`twos_complement`, and it is owed rather than defaulted: a data type added +later has to decide, because either default would answer for it silently +and one of them accepts a cast the spec does not define. Registration +refuses a data type that has not decided. + +Entities gain `name`, the spelling a document writes, as distinct from +`identifier`, the key they are tabled under. The two differ only for the +raw-bytes family, whose identifier is invented and belongs in no message +a reader sees; a message about an `r24` now says `r24`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py index f9bdc58186..3632740fd4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_entity.py @@ -805,6 +805,19 @@ def accepts(cls, name: str) -> bool: """ return name == cls.identifier + @property + def name(self) -> str: + """The name this entity carries, as a document writes it. + + The identifier, except for a family whose name carries a value: + the raw-bytes family's identifier is invented and belongs in no + message a reader sees, and its name is the `r24` the document + wrote. Anything a reader sees wants this; anything looking a + class up wants `identifier`. + """ + from_name = _plan(type(self)).from_name + return type(self).identifier if from_name is None else cast("str", getattr(self, from_name)) + @classmethod def create_unchecked(cls, **fields: object) -> Self: """This entity with these fields, built without the constructor's checks. @@ -1041,6 +1054,16 @@ class DataTypeEntity(MetadataEntity): scalar_storage: ClassVar[StorageClass] + twos_complement: ClassVar[bool] + """Whether this type's scalars are two's complement integers. + + Asked by `cast_value`, whose `out_of_range: "wrap"` is defined only + for such a target. Owed rather than defaulted: a data type added + later must decide, because either default would answer for it + silently -- and getting it wrong in one direction accepts a cast the + spec does not define. + """ + def storage_class(self) -> StorageClass | None: """How one scalar occupies bytes, or None if undetermined. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index b917e3eb9f..4b676c5202 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -113,7 +113,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb if not isinstance(data_type, DataTypeEntity): return () storage = data_type.storage_class() - name = type(data_type).identifier + name = data_type.name if storage == "variable_length": return problem( (), diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 3ac7b4de15..5571b2cc2a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -5,12 +5,13 @@ """ from dataclasses import dataclass -from typing import ClassVar, Final, Literal, NotRequired, Self +from typing import TYPE_CHECKING, ClassVar, Final, Literal, NotRequired, Self from typing_extensions import TypedDict from zarr_metadata._common import JSONValue from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ValidationProblem from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON from zarr_metadata.v3._entity import ( ArrayArrayCodec, @@ -20,6 +21,9 @@ ) from zarr_metadata.v3._parts import ArrayParts +if TYPE_CHECKING: + from collections.abc import Iterator + CAST_VALUE_CODEC_NAME: Final = "cast_value" """The `name` field value of the `cast_value` codec.""" @@ -135,6 +139,27 @@ class CastValueOptions(Configuration): out_of_range: CastOutOfRangeMode | UNSET = UNSET scalar_map: ScalarMap | UNSET = UNSET + def problems(self) -> "Iterator[ValidationProblem]": + """`out_of_range: "wrap"` needs a target that wraps. + + The spec defines wrapping only for integral targets with a two's + complement representation, which is a fact each data type states + about itself. A target out of scope is not judged: it may well be + an integral extension type, and judging it here would be guessing. + """ + target = self.data_type + if ( + self.out_of_range == "wrap" + and isinstance(target, DataTypeEntity) + and not type(target).twos_complement + ): + yield ValidationProblem( + ("out_of_range",), + "out_of_range 'wrap' requires a two's complement integer data_type, " + f"got {target.name!r}", + "invalid_value", + ) + @dataclass(frozen=True) class CastValueCodec(ArrayArrayCodec): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 9697aeb7cb..b67ce0eaab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -132,7 +132,7 @@ def incoming_problems(self, incoming: ArrayParts | None) -> tuple[ValidationProb return problem( (), "scale_offset is defined for integer and floating-point data types, not " - f"{type(data_type).identifier!r}", + f"{data_type.name!r}", "invalid_value", ) return tuple( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py index 2ee9cd355d..c65421c506 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/_families.py @@ -71,6 +71,7 @@ class IntegerDataType(DataTypeEntity): configuration: Configuration = field(default_factory=Configuration) bounds: ClassVar[tuple[int, int]] + twos_complement: ClassVar[bool] = True def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: low, high = type(self).bounds @@ -90,6 +91,7 @@ class FloatDataType(DataTypeEntity): configuration: Configuration = field(default_factory=Configuration) scalar_storage: ClassVar[StorageClass] = "multi_byte" + twos_complement: ClassVar[bool] = False hex_parser: ClassVar[Callable[[str], object]] largest: ClassVar[float | None] @@ -132,6 +134,7 @@ class ComplexDataType(DataTypeEntity): configuration: Configuration = field(default_factory=Configuration) scalar_storage: ClassVar[StorageClass] = "multi_byte" + twos_complement: ClassVar[bool] = False component: ClassVar[type[FloatDataType]] def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: @@ -203,6 +206,9 @@ class NumpyTimeDataType(DataTypeEntity): configuration: NumpyTimeOptions scalar_storage: ClassVar[StorageClass] = "multi_byte" + # Stored as a signed integer, but it denotes an instant or a + # duration; wrapping one is not a defined cast. + twos_complement: ClassVar[bool] = False def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: if value == "NaT": diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py index 3bb762b578..958fc0c795 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bool.py @@ -41,6 +41,7 @@ class BoolDataType(DataTypeEntity): configuration: Configuration = field(default_factory=Configuration) scalar_storage: ClassVar[StorageClass] = "single_byte" + twos_complement: ClassVar[bool] = False identifier: ClassVar[str] = BOOL_DATA_TYPE_NAME def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index d89c09e1b3..3f06346d02 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -67,6 +67,7 @@ class BytesDataType(DataTypeEntity): configuration: Configuration = field(default_factory=Configuration) scalar_storage: ClassVar[StorageClass] = "variable_length" + twos_complement: ClassVar[bool] = False identifier: ClassVar[str] = BYTES_DATA_TYPE_NAME def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 86158d3737..f7f89d03c8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -106,6 +106,7 @@ class RawBytesDataType(DataTypeEntity): """The spelling as written -- `r8`, `r008` -- which is where the width lives.""" scalar_storage: ClassVar[StorageClass] = "single_byte" + twos_complement: ClassVar[bool] = False identifier: ClassVar[str] = RAW_BYTES_FAMILY @classmethod diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 02ca4a7dcd..79979410e4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -41,6 +41,7 @@ class StringDataType(DataTypeEntity): configuration: Configuration = field(default_factory=Configuration) scalar_storage: ClassVar[StorageClass] = "variable_length" + twos_complement: ClassVar[bool] = False identifier: ClassVar[str] = STRING_DATA_TYPE_NAME def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 41bf37ab40..977a12d22f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -154,6 +154,7 @@ class StructDataType(DataTypeEntity): identifier: ClassVar[str] = STRUCT_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "single_byte" + twos_complement: ClassVar[bool] = False def canonical(self) -> Self: """Each field's data type in its own canonical form.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py index fe0978b2ed..c3f61676d5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/entity.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/entity.py @@ -196,9 +196,12 @@ class AcmeLz4Codec(BytesBytesCodec): endianness is needed), and `fill_value_problems(value, loc) -> tuple[ValidationProblem, ...]`, abstract: it judges a document's `fill_value`, and a type that accepts any says so with `return ()`. - These composition hooks return tuples; a record's `problems` yields. - The families `IntegerDataType`, `FloatDataType`, `ComplexDataType` and - `NumpyTimeDataType` carry both for the types they cover; a family of + It also declares `twos_complement`, whether its scalars are two's + complement integers, which `cast_value` asks before it lets a cast + wrap. These composition hooks return tuples; a record's `problems` + yields. The families `IntegerDataType`, `FloatDataType`, + `ComplexDataType` and `NumpyTimeDataType` carry all three for the + types they cover; a family of your own is a plain subclass that is never registered itself, and passes its class variables down. - A chunk grid: `grid(array_shape)`, abstract, and `shape_problems`; see diff --git a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py index 9495f9b6aa..1c302f2629 100644 --- a/packages/zarr-metadata/tests/rules/test_v3_array_rules.py +++ b/packages/zarr-metadata/tests/rules/test_v3_array_rules.py @@ -854,3 +854,52 @@ def test_error_scale_offset_data_type_has_no_arithmetic() -> None: ("codecs", 0), "scale_offset is defined for integer and floating-point data types, not 'bool'", ) + + +# (a cast target, whether `out_of_range: "wrap"` is defined for it) +WRAP_TARGETS: dict[str, tuple[object, bool]] = { + "int32": ("int32", True), + "uint64": ("uint64", True), + "bool": ("bool", False), + "float32": ("float32", False), + "complex64": ("complex64", False), + "raw-bytes": ("r8", False), + "string": ("string", False), + "numpy-time": ( + {"name": "numpy.datetime64", "configuration": {"unit": "s", "scale_factor": 1}}, + False, + ), + # Out of scope, so it may be an integral extension type; declining + # beats guessing. + "unmodelled": ("mycorp.bigint", True), +} + + +@pytest.mark.parametrize(("target", "allowed"), WRAP_TARGETS.values(), ids=list(WRAP_TARGETS)) +def test_wrap_requires_a_twos_complement_integer_target(target: object, allowed: bool) -> None: + document = { + **BASE, + "codecs": ( + {"name": "cast_value", "configuration": {"data_type": target, "out_of_range": "wrap"}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ), + } + wrap = [ + problem + for problem in validate_array_metadata_v3(document) + if problem.loc == ("codecs", 0, "configuration", "out_of_range") + ] + assert (len(wrap) == 0) is allowed, wrap + + +def test_the_wrap_message_names_the_spelling_not_the_family() -> None: + # `r` is an invented lookup key, not a name any document writes. + document = { + **BASE, + "codecs": ( + {"name": "cast_value", "configuration": {"data_type": "r24", "out_of_range": "wrap"}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + ), + } + messages = [problem.message for problem in validate_array_metadata_v3(document)] + assert any("got 'r24'" in message for message in messages), messages diff --git a/packages/zarr-metadata/tests/v3/test_acme_decimal.py b/packages/zarr-metadata/tests/v3/test_acme_decimal.py index 996b98b0d9..09c5fc592b 100644 --- a/packages/zarr-metadata/tests/v3/test_acme_decimal.py +++ b/packages/zarr-metadata/tests/v3/test_acme_decimal.py @@ -104,6 +104,7 @@ class AcmeDecimalDataType(DataTypeEntity): identifier: ClassVar[str] = ACME_DECIMAL_DATA_TYPE_NAME scalar_storage: ClassVar[StorageClass] = "multi_byte" + twos_complement: ClassVar[bool] = False def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: """A decimal literal whose digits fit `precision` and `scale`. diff --git a/packages/zarr-metadata/tests/v3/test_extension_api.py b/packages/zarr-metadata/tests/v3/test_extension_api.py index 420ba9b606..5267bd5f34 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_api.py +++ b/packages/zarr-metadata/tests/v3/test_extension_api.py @@ -82,6 +82,7 @@ class AcmeFloat8DataType(DataTypeEntity): identifier: ClassVar[str] = "acme.float8" scalar_storage: ClassVar[StorageClass] = "single_byte" + twos_complement: ClassVar[bool] = False def fill_value_problems(self, value: object, loc: Loc = ()) -> tuple[ValidationProblem, ...]: return () @@ -301,6 +302,7 @@ class AcmeFixedDataType(DataTypeEntity): identifier: ClassVar[str] = "acme.fixed" scalar_storage: ClassVar[StorageClass] = "multi_byte" + twos_complement: ClassVar[bool] = False @classmethod def name_problems(cls, name: str) -> Iterator[ValidationProblem]: @@ -813,11 +815,30 @@ def test_error_a_data_type_judges_its_fill_values() -> None: class Lax(DataTypeEntity): identifier: ClassVar[str] = "acme.lax" scalar_storage: ClassVar[StorageClass] = "single_byte" + twos_complement: ClassVar[bool] = False with pytest.raises(TypeError, match="does not define fill_value_problems"): CORE_AND_EXTENSIONS.extended_with(Lax) +def test_error_a_data_type_must_say_whether_it_wraps() -> None: + # `cast_value`'s `out_of_range: "wrap"` is defined only for two's + # complement integers. Defaulting either way would answer for a new + # data type silently, and one direction accepts an undefined cast. + @dataclass(frozen=True) + class Undecided(DataTypeEntity): + identifier: ClassVar[str] = "acme.undecided" + scalar_storage: ClassVar[StorageClass] = "single_byte" + + def fill_value_problems( + self, value: object, loc: Loc = () + ) -> tuple[ValidationProblem, ...]: + return () + + with pytest.raises(TypeError, match="does not declare twos_complement"): + CORE_AND_EXTENSIONS.extended_with(Undecided) + + def test_error_a_list_of_problem_tuples_is_refused() -> None: # `problem()` returns a one-element tuple; a list of those would pass # the constructor and fail inside `coerce`, far from the mistake.