From bb5d3bf5d1b1b8164329d7d19edcd646809b8305 Mon Sep 17 00:00:00 2001 From: CarloLucibello Date: Fri, 3 Jul 2026 07:57:59 +0200 Subject: [PATCH 1/2] Add `Features`/`ClassLabel` schema views (review item 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give a dataset's schema a Julian home. `ds.features` now returns a `Features` view (an `AbstractDict{String, Any}`) instead of a raw `Py` mapping: indexing a column yields a wrapped `ClassLabel`/`Value` leaf (other feature types stay raw `Py`), each forwarding attribute/method access to Python (`cl.names`, `cl.num_classes`, `cl.int2str(i)`, `cl.str2int(s)`, `v.dtype`). The views are `Py`-backed and can be built from Julia (`ClassLabel(names=[…])`, `Value("int64")`, `Features(Dict(…))`) and handed back to Python via `jl2py` (e.g. a `features=` schema argument). Public-but-unexported Julian conveniences decode labels in one call from a dataset + column: `class_names(ds, col)`, `int2str(ds, col, i)`, `str2int(ds, col, s)`. Class ids are 0-based *data* and pass through with no index offset; the 0→1 bridge only appears when indexing the 1-based Julia `names` vector (`names[ids .+ 1]`), as documented. Handled at the access site (a `:features` branch in `Dataset`'s `getproperty`), so the `py2jl` batch hot path is untouched — `Features` subclasses `dict` and would otherwise tax every batch observation. `Features` caches its column names at construction so `keys`/`length`/iteration never call Python (safe from the REPL's async completion, mirroring `DatasetDict`). `Features`/`ClassLabel` are exported; `Value` and the decode helpers are public but unexported — the Pythonic idioms (`ds.features`, method chaining, `datasets.ClassLabel(…)`) are the primary interface. Tests: test/features.jl (47 assertions, all local/CI-safe). Docs: API entries, a "Schema: features and labels" guide section, CHANGELOG + AGENTS.md notes. Co-Authored-By: Claude Opus 4.8 (1M context) --- AGENTS.md | 8 ++ CHANGELOG.md | 12 ++ docs/make.jl | 7 +- docs/src/api.md | 12 ++ docs/src/guide.md | 84 +++++++++++ src/HuggingFaceDatasets.jl | 8 +- src/dataset.jl | 3 + src/features.jl | 279 +++++++++++++++++++++++++++++++++++++ src/transforms.jl | 2 + test/features.jl | 106 ++++++++++++++ test/runtests.jl | 4 + 11 files changed, 520 insertions(+), 5 deletions(-) create mode 100644 src/features.jl create mode 100644 test/features.jl diff --git a/AGENTS.md b/AGENTS.md index e738087..5e39eb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,14 @@ that lazily converts observations to Julia types). PIL images into Julia types; `jl2py` is the write-path dual. The `"julia"` format is numpy-backed, so numeric array columns decode to real N-D Julia arrays and image columns decode to raw numeric arrays (not `Colorant` colorviews). +- `src/features.jl` — `Py`-backed views over a dataset's schema: `Features` (an + `AbstractDict` returned by `ds.features`), and the `ClassLabel`/`Value` leaves it + wraps, each forwarding attribute/method access to Python (`cl.names`, + `cl.int2str`, `v.dtype`). Handled at the access site (a `:features` branch in + `Dataset`'s `getproperty`), never in the `py2jl` batch hot path. Also the Julian + label-decoding helpers `class_names`/`int2str`/`str2int` (`(ds, col, …)`), and + `jl2py` overloads so a Julia-built schema round-trips into a `features=` argument. + Everything here is public but unexported; the Pythonic idioms are primary. - `src/serialization.jl` — `Serialization.serialize`/`deserialize` for `Dataset`, so it can cross a process boundary (process-parallel data loaders). Never serializes the wrapped `Py` directly; instead uses `datasets`' own pickle diff --git a/CHANGELOG.md b/CHANGELOG.md index fb128eb..ed1e7d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,18 @@ Julia values instead of raw Python objects. See **Breaking** below before upgrad `BoundsError`, instead of `AssertionError` — update any code catching `AssertionError`. ### Added +- Julia views over a dataset's schema. `ds.features` now returns a `Features` view (an + `AbstractDict{String, Any}`) instead of a raw `Py` mapping: indexing a column yields a wrapped + `ClassLabel`/`Value` leaf (other feature types stay raw `Py`), each forwarding attribute and + method access to Python (`cl.names`, `cl.num_classes`, `cl.int2str(i)`, `cl.str2int(s)`, + `v.dtype`). The views can be built from Julia (`ClassLabel(names=[…])`, `Value("int64")`, + `Features(Dict(…))`) and handed back to Python via `jl2py` (e.g. a `features=` schema + argument). Public but unexported Julian conveniences look up a column's `ClassLabel` in one + call: `class_names(ds, col)`, `int2str(ds, col, i)`, `str2int(ds, col, s)` (label ids are + 0-based data, passed through with no index offset), plus `features(ds)` as the function form + of `ds.features`. `Features`/`ClassLabel`/`Value` and these helpers are all public but not + exported — the Pythonic idioms (`ds.features`, method chaining, `datasets.ClassLabel(…)`) are + the primary interface. - `Serialization` support for `Dataset`: `Serialization.serialize`/`deserialize` now work, so a `Dataset` can cross a process boundary — the prerequisite for process-parallel data loaders (e.g. a `MLUtils.DataLoader(ds; num_workers=N)` that spreads `getobs` over worker diff --git a/docs/make.jl b/docs/make.jl index 48e6d0b..1df8c83 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,8 +1,13 @@ using HuggingFaceDatasets using Documenter +# Bring the public-but-unexported schema conveniences into doctest scope, and disable +# `datasets`' tqdm progress bars so doctests that trigger them (e.g. `class_encode_column`) +# don't emit a progress line into the captured output. DocMeta.setdocmeta!(HuggingFaceDatasets, :DocTestSetup, - :(using HuggingFaceDatasets, PythonCall); recursive=true) + :(using HuggingFaceDatasets, PythonCall; + using HuggingFaceDatasets: features, class_names, int2str, str2int, Value; + HuggingFaceDatasets.datasets.disable_progress_bars()); recursive=true) makedocs(; modules=[HuggingFaceDatasets], diff --git a/docs/src/api.md b/docs/src/api.md index 95fe7a6..63fa0f3 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -18,6 +18,18 @@ IterableDatasetDict Column ``` +## Schema (features) + +```@docs +features +Features +ClassLabel +Value +class_names +int2str +str2int +``` + ## Loading ```@docs diff --git a/docs/src/guide.md b/docs/src/guide.md index 3f2fe3c..527e158 100644 --- a/docs/src/guide.md +++ b/docs/src/guide.md @@ -2,6 +2,8 @@ CurrentModule = HuggingFaceDatasets DocTestSetup = quote using HuggingFaceDatasets, PythonCall + using HuggingFaceDatasets: features, class_names, int2str, str2int, Value + HuggingFaceDatasets.datasets.disable_progress_bars() end ``` @@ -128,6 +130,88 @@ Keyword arguments are forwarded as Python keyword arguments, so calls like [`datasets` documentation](https://huggingface.co/docs/datasets) for the exact meaning of each method's arguments. +## Inspecting the schema: features and labels + +Every dataset carries a **schema** describing each column's type. `ds.features` returns it as +a [`Features`](@ref) view — an `AbstractDict` from column name to feature type — so you can +inspect dtypes and, crucially, decode integer class labels. Indexing a column yields its +feature: a [`Value`](@ref) for a scalar column (carrying an Arrow `dtype`), a +[`ClassLabel`](@ref) for an encoded label, and so on. + +```jldoctest guide +julia> ds = Dataset((; text=["good", "bad", "good"], label=["pos", "neg", "pos"])); + +julia> ds.features +{'text': Value('string'), 'label': Value('string')} + +julia> ds.features["text"] +Value('string') +``` + +The most useful leaf is [`ClassLabel`](@ref), which maps integer class ids to names. Turn a +string column into one with the forwarded `class_encode_column`, then read the mapping straight +off the feature with Pythonic method chaining (`.names`, `.int2str`, `.str2int`): + +```jldoctest guide +julia> ds = Dataset((; label=["pos", "neg", "pos"])); + +julia> ds = ds.class_encode_column("label"); # string column -> ClassLabel (names sorted) + +julia> cl = ds.features["label"] +ClassLabel(names=['neg', 'pos']) + +julia> cl.names +2-element Vector{String}: + "neg" + "pos" + +julia> cl.int2str(0) # 0-based class id -> name (no index offset) +"neg" + +julia> cl.str2int("pos") +1 + +julia> ds["label"] # the stored ids are 0-based data, not 1-based indices +3-element HuggingFaceDatasets.Column{Int64}: + 1 + 0 + 1 +``` + +!!! note "Class ids are 0-based data" + A `ClassLabel` column stores **0-based class ids** (`ds["label"]` above is `[1, 0, 1]`), + not 1-based Julia indices. `int2str`/`str2int` pass ids through to Python unchanged; only + the wrapper's `getindex`/iteration interface is 1-based. Decoding a whole column is + therefore `cl.names[ds["label"] .+ 1]`, where the `+1` bridges a 0-based id to a 1-based + Julia position. + +For the common "from a dataset and column name" case there are also public (unexported) Julian +shortcuts — [`class_names`](@ref), [`int2str`](@ref), and [`str2int`](@ref) — that look up the +column's `ClassLabel` for you (and error clearly if it isn't one): + +```jldoctest guide +julia> ds = Dataset((; label=["pos", "neg", "pos"])); + +julia> ds = ds.class_encode_column("label"); + +julia> class_names(ds, "label") +2-element Vector{String}: + "neg" + "pos" + +julia> int2str(ds, "label", [0, 1, 1]) # decode a batch of ids in one call +3-element Vector{String}: + "neg" + "pos" + "pos" +``` + +Reach these as `HuggingFaceDatasets.class_names` etc., or bring them into scope with +`using HuggingFaceDatasets: class_names, int2str, str2int`. To **construct** a schema from +Julia — e.g. to pass as a `features=` argument — build the wrapper types (also public but +unexported: `HuggingFaceDatasets.ClassLabel(names=["neg", "pos"])`, +`HuggingFaceDatasets.Value("int64")`) and hand them back to Python with [`jl2py`](@ref). + ## The `"julia"` format and transforms Datasets are returned in the `"julia"` format by default, so indexing yields native Julia diff --git a/src/HuggingFaceDatasets.jl b/src/HuggingFaceDatasets.jl index 7c941b5..f9b14da 100644 --- a/src/HuggingFaceDatasets.jl +++ b/src/HuggingFaceDatasets.jl @@ -45,6 +45,10 @@ export py2jl, jl2numpy, numpy2jl +include("features.jl") +export Features, ClassLabel +@compat public Value, features, class_names, int2str, str2int + include("load_dataset.jl") export load_dataset @@ -53,12 +57,8 @@ export concatenate_datasets, interleave_datasets, load_from_disk -# Recipe-based `Serialization` for `Dataset` (ships an on-disk path, never a `Py`), so a -# `Dataset` can be sent to `Distributed` worker processes — the basis for process-parallel -# data loaders. Included after `toplevel.jl` as it uses `load_from_disk`. include("serialization.jl") -# `public` is a Julia 1.11+ keyword; `@compat` makes it a no-op on the supported 1.10. @compat public from_csv, from_json, from_parquet function __init__() diff --git a/src/dataset.jl b/src/dataset.jl index f45132b..2eff495 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -224,6 +224,9 @@ function Base.getproperty(ds::Dataset, s::Symbol) if s in fieldnames(Dataset) return getfield(ds, s) end + # Return the schema as a Julia `Features` view (see `features.jl`) instead of the raw + # Python mapping, so `ds.features["label"]` yields a wrapped `ClassLabel`/`Value` leaf. + s === :features && return features(ds) # Route the format and `map`/`filter` methods to this package's own versions (see # `_method_override`); every other name forwards to the wrapped Python object. override = _method_override(ds, s) diff --git a/src/features.jl b/src/features.jl new file mode 100644 index 0000000..f72c829 --- /dev/null +++ b/src/features.jl @@ -0,0 +1,279 @@ +# A Julia view over a dataset's schema: `datasets.Features` and its leaf feature types +# (`ClassLabel`, `Value`). These are the schema objects returned by `ds.features`; they are +# NOT row data, so they are handled at the access site (`Dataset`'s `getproperty`, and the +# `features`/`class_names`/`int2str`/`str2int` helpers) and never touched by the generic +# `py2jl` batch hot path. +# +# Each type is `Py`-backed and forwards attribute/method access to the wrapped Python object +# (mirroring `Dataset`/`Column`), so the full Python surface (`.names`, `.num_classes`, +# `.int2str`, `.dtype`, ...) stays reachable under the same names, with results re-wrapped by +# `py2jl`. The matching `jl2py` overloads (in `transforms.jl`) unwrap them back to `.py`, so a +# view built or fetched in Julia can be handed straight back to Python (e.g. a `features=` +# schema argument). + +""" + ClassLabel(; names, num_classes) + +A Julia view over a `datasets.ClassLabel` feature: the integer-encoded label type whose +`names` map class ids to human-readable strings. + +Construct one from Julia (`ClassLabel(names=["neg", "pos"])`) — forwarding to +`datasets.ClassLabel` — or obtain one from a dataset's schema via `ds.features["label"]` (see +[`features`](@ref)). Attribute and method access forwards to Python, so `cl.names`, +`cl.num_classes`, `cl.int2str(i)`, and `cl.str2int(s)` all work, with results converted by +[`py2jl`](@ref). + +Label integers are **0-based class ids** (data, not 1-based Julia indices): `int2str`/`str2int` +pass them through to Python unchanged. See also [`class_names`](@ref), [`int2str`](@ref), +[`str2int`](@ref), and [`Features`](@ref). + +# Examples + +```jldoctest +julia> ds = Dataset((; label=["cat", "dog", "dog"], x=[1, 2, 3])); + +julia> ds = ds.class_encode_column("label"); # string column -> ClassLabel + +julia> cl = ds.features["label"] +ClassLabel(names=['cat', 'dog']) + +julia> cl.names +2-element Vector{String}: + "cat" + "dog" + +julia> cl.int2str(1) # 0-based class id -> name +"dog" + +julia> cl.str2int("cat") +0 +``` +""" +struct ClassLabel + py::Py +end + +function ClassLabel(; names = nothing, num_classes = nothing) + # Convert `names` to a Python list: a Julia `Vector` passed straight through as a kwarg + # stays an unconverted Julia object, which later breaks `datasets`' JSON schema encoding. + names === nothing || (names = jl2py(names)) + return ClassLabel(datasets.ClassLabel(; names, num_classes)) +end + +""" + Value(dtype::AbstractString) + +A Julia view over a `datasets.Value` feature: a scalar column type carrying an Arrow +`dtype` (e.g. `"int64"`, `"float32"`, `"string"`). Construct one with `Value("int64")` +(forwarding to `datasets.Value`) or obtain it from a schema via [`features`](@ref). Attribute +access forwards to Python, so `v.dtype` returns the dtype string (e.g. `ds.features["x"]`). + +See also [`Features`](@ref) and [`ClassLabel`](@ref). +""" +struct Value + py::Py +end + +Value(dtype::AbstractString) = Value(datasets.Value(dtype)) + +# Forward attribute/method access on a leaf view to the wrapped Python object, re-wrapping the +# result with `py2jl` (callables become a `CallableWrapper`, like `Dataset`'s `getproperty`). +for T in (:ClassLabel, :Value) + @eval function Base.getproperty(x::$T, s::Symbol) + s === :py && return getfield(x, :py) + res = getproperty(getfield(x, :py), s) + return pycallable(res) ? CallableWrapper(res) : py2jl(res) + end + @eval Base.show(io::IO, x::$T) = print(io, getfield(x, :py)) +end + +""" + Features(schema::AbstractDict) + +A Julia view over a `datasets.Features` schema: an ordered mapping from column name to its +feature type. `Features <: AbstractDict{String, Any}`, so it indexes, iterates, and supports +`keys`/`values`/`haskey`/`get` like a dict; indexing a column returns the wrapped leaf +([`ClassLabel`](@ref), [`Value`](@ref)) when recognized, or the raw `Py` otherwise (nested +features, `Image`, `Audio`, `Sequence`, ...). + +Obtained from a dataset via `ds.features` (or the [`features`](@ref) function), or built from +Julia (`Features(Dict("label" => ClassLabel(names=["neg", "pos"])))`) and passed back to Python +as a `features=` schema argument. + +The column names are cached at construction, so `keys`/`length`/iteration never call Python +(safe from the REPL's async `feat[` completion, mirroring [`DatasetDict`](@ref)). + +# Examples + +```jldoctest +julia> ds = Dataset((; label=[0, 1, 1], x=[1.0, 2.0, 3.0])); + +julia> f = ds.features; + +julia> collect(keys(f)) +2-element Vector{String}: + "label" + "x" + +julia> f["x"] +Value('float64') +``` +""" +struct Features <: AbstractDict{String, Any} + py::Py + names::Vector{String} # cached, ordered column names (Python-free keys/length/iteration) +end + +function Features(py::Py) + pyisinstance(py, datasets.Features) || + throw(ArgumentError("expected a `datasets.Features`, got $(pytype(py))")) + return Features(py, String[pyconvert(String, k) for k in py.keys()]) +end + +Features(schema::AbstractDict) = Features(datasets.Features(jl2py(schema))) + +# Wrap a schema leaf: recognized feature types get a Julia view; everything else (nested +# `Features`, `Sequence`, `Image`, `Audio`, ...) is left as a raw `Py` on purpose. +function _wrap_feature(x::Py) + if pyisinstance(x, datasets.ClassLabel) + return ClassLabel(x) + elseif pyisinstance(x, datasets.Value) + return Value(x) + elseif pyisinstance(x, datasets.Features) + return Features(x) + else + return x + end +end + +Base.getindex(f::Features, k::AbstractString) = _wrap_feature(getfield(f, :py)[k]) +Base.getindex(f::Features, k::Symbol) = f[string(k)] + +# `keys`/`length`/`haskey`/iteration answer from the cached `names` (no Python call). +Base.keys(f::Features) = getfield(f, :names) +Base.length(f::Features) = length(getfield(f, :names)) +Base.haskey(f::Features, k) = string(k) in getfield(f, :names) + +function Base.iterate(f::Features, state = 1) + names = getfield(f, :names) + state > length(names) && return nothing + k = names[state] + return (k => f[k], state + 1) +end + +# Show the Python schema repr (like `Dataset`/`DatasetDict`) rather than the generic +# `AbstractDict` multi-line display. +Base.show(io::IO, f::Features) = print(io, getfield(f, :py)) +Base.show(io::IO, ::MIME"text/plain", f::Features) = print(io, getfield(f, :py)) + +""" + features(ds::Dataset) + +Return the schema of `ds` as a [`Features`](@ref) view (also reachable as `ds.features`). +Indexing a column yields its feature type, with [`ClassLabel`](@ref)/[`Value`](@ref) leaves +wrapped for Julian access. + +See also [`class_names`](@ref), [`int2str`](@ref), and [`str2int`](@ref). + +# Examples + +```jldoctest +julia> ds = Dataset((; label=[0, 1, 1], x=[1, 2, 3])); + +julia> features(ds)["label"] +Value('int64') +``` +""" +features(ds::Dataset) = Features(getfield(ds, :py).features) + +# The `ClassLabel` for column `col`, erroring clearly when the column is not one. +function _classlabel(ds::Dataset, col) + f = features(ds)[string(col)] + f isa ClassLabel && return f + throw(ArgumentError("column \"$col\" is not a ClassLabel feature; got $(_featkind(f))")) +end + +_featkind(f::Value) = "a Value feature" +_featkind(f) = "$(typeof(f))" + +""" + class_names(ds::Dataset, col) + +The ordered class names of column `col`'s [`ClassLabel`](@ref) feature, as a `Vector{String}`; +`names[i]` is the name of class id `i - 1` (ids are 0-based). Errors if `col` is not a +`ClassLabel`. Equivalent to the Pythonic `ds.features[col].names`. + +See also [`int2str`](@ref), [`str2int`](@ref), and [`features`](@ref). + +# Examples + +```jldoctest +julia> ds = Dataset((; label=["cat", "dog", "dog"])); + +julia> ds = ds.class_encode_column("label"); + +julia> class_names(ds, "label") +2-element Vector{String}: + "cat" + "dog" +``` +""" +class_names(ds::Dataset, col) = pyconvert(Vector{String}, _classlabel(ds, col).py.names) + +""" + int2str(ds::Dataset, col, i) + +Decode 0-based class id(s) `i` (an integer or a vector of integers) to class name(s) via +column `col`'s [`ClassLabel`](@ref), so **no index offset is applied**. Errors if `col` is not +a `ClassLabel`. Equivalent to the Pythonic `ds.features[col].int2str(i)`. + +See also [`str2int`](@ref), [`class_names`](@ref), and [`features`](@ref). + +# Examples + +```jldoctest +julia> ds = Dataset((; label=["cat", "dog", "dog"])); + +julia> ds = ds.class_encode_column("label"); + +julia> int2str(ds, "label", 1) +"dog" + +julia> int2str(ds, "label", [0, 1, 1]) +3-element Vector{String}: + "cat" + "dog" + "dog" +``` +""" +int2str(ds::Dataset, col, i) = py2jl(_classlabel(ds, col).py.int2str(jl2py(i))) + +""" + str2int(ds::Dataset, col, s) + +Encode class name(s) `s` (a string or a vector of strings) to their 0-based class id(s) via +column `col`'s [`ClassLabel`](@ref). Errors if `col` is not a `ClassLabel`. Equivalent to the +Pythonic `ds.features[col].str2int(s)`. + +See also [`int2str`](@ref), [`class_names`](@ref), and [`features`](@ref). + +# Examples + +```jldoctest +julia> ds = Dataset((; label=["cat", "dog", "dog"])); + +julia> ds = ds.class_encode_column("label"); + +julia> str2int(ds, "label", "dog") +1 +``` +""" +str2int(ds::Dataset, col, s) = py2jl(_classlabel(ds, col).py.str2int(jl2py(s))) + +# `jl2py` for the schema views: unwrap back to the underlying Python object so a +# `Features`/`ClassLabel`/`Value` built or fetched in Julia can be handed back to Python +# (e.g. a `features=` schema argument). Defined here, after the types; `jl2py`'s other methods +# live in `transforms.jl`. +jl2py(x::Features) = getfield(x, :py) +jl2py(x::ClassLabel) = getfield(x, :py) +jl2py(x::Value) = getfield(x, :py) diff --git a/src/transforms.jl b/src/transforms.jl index ffaa0bd..0460be7 100644 --- a/src/transforms.jl +++ b/src/transforms.jl @@ -187,6 +187,8 @@ jl2py(x::DatasetDict) = getfield(x, :py) jl2py(x::IterableDataset) = getfield(x, :py) jl2py(x::IterableDatasetDict) = getfield(x, :py) jl2py(x::Column) = getfield(x, :py) +# `jl2py` for the schema views (`Features`/`ClassLabel`/`Value`) is defined in `features.jl`, +# where those types are declared. function jl2py(x::AbstractDict) d = pydict() diff --git a/test/features.jl b/test/features.jl new file mode 100644 index 0000000..10549bf --- /dev/null +++ b/test/features.jl @@ -0,0 +1,106 @@ +# Schema views (`Features`/`ClassLabel`/`Value`) and the `class_names`/`int2str`/`str2int` +# helpers. All local (no network), so these run in CI. `Features`/`ClassLabel` are exported +# (via `runtests.jl`'s `using`); the rest are public but unexported, so bring them into scope. +using HuggingFaceDatasets: Value, features, class_names, int2str, str2int + +# A tiny classification dataset: `class_encode_column` turns the string column into a +# `ClassLabel` (names sorted: "cat" => 0, "dog" => 1). +_encoded() = Dataset((; label = ["cat", "dog", "dog", "cat"], x = [10, 20, 30, 40])).class_encode_column("label") + +@testset "Features view" begin + ds = _encoded() + + f = features(ds) + @test f isa Features + @test f isa AbstractDict + @test ds.features isa Features # `ds.features` getproperty branch, not raw Py + + @test Set(keys(f)) == Set(["label", "x"]) + @test length(f) == 2 + @test haskey(f, "label") + @test haskey(f, :label) # Symbol key + @test !haskey(f, "missing") + + # iteration yields name => feature pairs (so `Dict(f)`, `collect`, ... work) + d = Dict(f) + @test Set(keys(d)) == Set(["label", "x"]) + @test d["label"] isa ClassLabel + @test d["x"] isa Value + + # keys/length are Python-free (answered from the cached names) — safe for tab-completion + @test keys(f) == ["label", "x"] +end + +@testset "ClassLabel view" begin + ds = _encoded() + cl = ds.features["label"] + @test cl isa ClassLabel + + # attribute/method access forwards to Python (the primary, Pythonic idiom), converted by + # py2jl — including vector args + @test cl.names == ["cat", "dog"] + @test cl.num_classes == 2 + @test cl.int2str(1) == "dog" + @test cl.int2str([0, 1, 1]) == ["cat", "dog", "dog"] + @test cl.str2int("cat") == 0 + + # Julian convenience functions on (ds, col) + @test class_names(ds, "label") == ["cat", "dog"] + @test class_names(ds, :label) == ["cat", "dog"] # Symbol column + + # 0-based class ids pass through with NO offset (labels are data, not Julia indices) + @test int2str(ds, "label", 0) == "cat" + @test int2str(ds, "label", 1) == "dog" + @test str2int(ds, "label", "dog") == 1 + + # vector arguments decode/encode a whole batch in one call + @test int2str(ds, "label", [0, 1, 1]) == ["cat", "dog", "dog"] + @test str2int(ds, "label", ["cat", "dog"]) == [0, 1] + + # decoding a whole column matches `names[labels .+ 1]` (the 0-based id -> 1-based bridge) + labels = collect(ds["label"]) + @test int2str(ds, "label", labels) == class_names(ds, "label")[labels .+ 1] +end + +@testset "Value view" begin + ds = Dataset((; x = [1, 2, 3], f = [1.0, 2.0, 3.0], s = ["a", "b", "c"])) + @test ds.features["x"] isa Value + @test ds.features["x"].dtype == "int64" + @test ds.features["f"].dtype == "float64" + @test ds.features["s"].dtype == "string" +end + +@testset "non-ClassLabel column errors clearly" begin + ds = Dataset((; x = [1, 2, 3])) + @test_throws ArgumentError class_names(ds, "x") + @test_throws ArgumentError int2str(ds, "x", 0) + @test_throws ArgumentError str2int(ds, "x", "a") +end + +@testset "construct schema from Julia + round-trip through features=" begin + cl = ClassLabel(names = ["neg", "pos"]) + @test cl isa ClassLabel + @test cl.names == ["neg", "pos"] + @test cl.num_classes == 2 + + v = Value("int64") + @test v isa Value + @test v.dtype == "int64" + + sch = Features(Dict("label" => cl, "x" => v)) + @test sch isa Features + @test sch["label"] isa ClassLabel + @test sch["x"] isa Value + + # jl2py unwraps the views back to the underlying Python objects + @test pyis(jl2py(cl), cl.py) + @test pyis(jl2py(sch), sch.py) + + # feed the Julia-built schema back into construction via `features=` + ds = Dataset.from_dict(Dict("label" => [0, 1, 1, 0], "x" => [1, 2, 3, 4]); + features = jl2py(sch)) + @test ds.features["label"] isa ClassLabel + @test class_names(ds, "label") == ["neg", "pos"] + @test int2str(ds, "label", 0) == "neg" + @test int2str(ds, "label", 1) == "pos" +end diff --git a/test/runtests.jl b/test/runtests.jl index 951aa7a..857108d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -12,6 +12,10 @@ end include("dataset.jl") end +@testset "features" begin + include("features.jl") +end + @testset "datasetdict" begin include("datasetdict.jl") end From 4d3ff8a8dba3f9008c62d9ac0745796436b88e0f Mon Sep 17 00:00:00 2001 From: CarloLucibello Date: Fri, 3 Jul 2026 10:05:08 +0200 Subject: [PATCH 2/2] Fix Features AbstractDict surface: add `get`, raise `KeyError` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review of the schema views found two gaps against the `AbstractDict` contract the `Features` docstring advertises: - `get(f, key, default)` threw `MethodError` — Julia has no generic `Base.get(::AbstractDict, …)` fallback. Add it (Python-free via cached `haskey`). - Indexing a missing column surfaced Python's `KeyError` as a `PyException`; guard `getindex` with `haskey` so it raises a Julia `KeyError` instead. Tests extended to cover `get` (present/absent) and the `KeyError` path (51 pass). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/features.jl | 10 +++++++++- test/features.jl | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/features.jl b/src/features.jl index f72c829..9015afb 100644 --- a/src/features.jl +++ b/src/features.jl @@ -146,7 +146,12 @@ function _wrap_feature(x::Py) end end -Base.getindex(f::Features, k::AbstractString) = _wrap_feature(getfield(f, :py)[k]) +function Base.getindex(f::Features, k::AbstractString) + # Guard with the (Python-free) `haskey` so a missing column raises a Julia `KeyError` + # rather than surfacing Python's `KeyError` as a `PyException`. + haskey(f, k) || throw(KeyError(k)) + return _wrap_feature(getfield(f, :py)[k]) +end Base.getindex(f::Features, k::Symbol) = f[string(k)] # `keys`/`length`/`haskey`/iteration answer from the cached `names` (no Python call). @@ -154,6 +159,9 @@ Base.keys(f::Features) = getfield(f, :names) Base.length(f::Features) = length(getfield(f, :names)) Base.haskey(f::Features, k) = string(k) in getfield(f, :names) +# `AbstractDict` has no generic `get`, so provide it (Python-free via the cached `haskey`). +Base.get(f::Features, k, default) = haskey(f, k) ? f[k] : default + function Base.iterate(f::Features, state = 1) names = getfield(f, :names) state > length(names) && return nothing diff --git a/test/features.jl b/test/features.jl index 10549bf..6d8e21a 100644 --- a/test/features.jl +++ b/test/features.jl @@ -27,6 +27,12 @@ _encoded() = Dataset((; label = ["cat", "dog", "dog", "cat"], x = [10, 20, 30, 4 @test d["label"] isa ClassLabel @test d["x"] isa Value + # the rest of the `AbstractDict` surface the docstring promises + @test values(f) |> collect |> length == 2 + @test get(f, "label", nothing) isa ClassLabel + @test get(f, "missing", :deflt) === :deflt + @test_throws KeyError f["missing"] # not a raw Python `PyException` + # keys/length are Python-free (answered from the cached names) — safe for tab-completion @test keys(f) == ["label", "x"] end