diff --git a/HISTORY.md b/HISTORY.md index 80f6e9d65..136fc30ec 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -28,7 +28,7 @@ Fixed model argument evaluation world-age errors when loading without a valid pr Thread-safe tilde assignments now throw an `ArgumentError` if any container is replaced, including same-type immutable values: `(a = zero(T),)` → `Vector{T}(undef, n)`. Allocate mutable storage with `::Type{T}=Float64`; use `Vector{Union{Missing,Real}}` for imputation under AD. See [#1521](https://github.com/TuringLang/DynamicPPL.jl/pull/1521). -`VarNamedTuple` membership resolves `begin` and `end` against the stored array. See [#1490](https://github.com/TuringLang/DynamicPPL.jl/pull/1490). +`VarNamedTuple` membership resolves `begin` and `end` against the stored array when it has a template. See [#1490](https://github.com/TuringLang/DynamicPPL.jl/pull/1490). `ComponentVector` properties, including nested fields and slices, use consistent indices for membership, retrieval, and updates. See [#1491](https://github.com/TuringLang/DynamicPPL.jl/pull/1491). @@ -46,6 +46,8 @@ Added `evaluate!!(model, context, vi)` to evaluate with an explicit leaf context `keys`, `haskey`, `length`, `values`, `isempty` and `is_transformed(vi)` on a `VarInfo` without a `VectorValueAccumulator` now throw an `ArgumentError` instead of reporting no variables. Record values with `VarInfo(VectorValueAccumulator(), DynamicPPL.default_accumulators()...)` in place of `VarInfo()`. +`VarNamedTuple` lookups and model evaluation throw `ArgumentError` for indices with a different number of dimensions, indices using `begin`, `end` or `:`, or Boolean masks of a different shape, in values set by index without a template; previously these could return the wrong element or be treated as missing. Supply a template or the whole array, or pass the `VarNamedTuple` itself rather than a `Dict` of its pairs ([#1544](https://github.com/TuringLang/DynamicPPL.jl/pull/1544)). + `marginalize` and the `DynamicPPLMarginalLogDensitiesExt` extension are removed in DynamicPPL 0.43. Users requiring the existing Turing/MLD integration can remain on DynamicPPL 0.42.x with a compatible Turing release—for example, Turing 0.49.0—and MarginalLogDensities 0.4.3–0.4.x. See the [0.42 marginalisation documentation](https://turinglang.org/DynamicPPL.jl/v0.42/api/#Marginalisation). On newer DynamicPPL versions, MLD’s generic API requires a manually constructed log-density adapter; no drop-in model-level replacement is currently available. Migration of the integration into MLD is proposed in [MLD #47](https://github.com/ElOceanografo/MarginalLogDensities.jl/pull/47). diff --git a/docs/src/conditionfix.md b/docs/src/conditionfix.md index 926cb553c..a5098b23a 100644 --- a/docs/src/conditionfix.md +++ b/docs/src/conditionfix.md @@ -152,7 +152,7 @@ end Note that in this case since the `VarNamedTuple` has no knowledge of the length or shape of `y`, DynamicPPL will assume that `y` is a `Base.Vector` of unknown length (hence the `GrowableArray` above). -This will work fine as long as `y` is indeed a `Base.Vector`. +This will work fine as long as `y` is indeed a `Base.Vector` and the model does not index it with `begin`, `end`, `:` or a Boolean mask. However, if you want to avoid this, you should provide the full shape of `y` when defining the `VarNamedTuple`: ```@example 1 diff --git a/src/chains.jl b/src/chains.jl index be092f18e..ecab4c3e4 100644 --- a/src/chains.jl +++ b/src/chains.jl @@ -418,17 +418,34 @@ function predict( return if include_all prediction else + # Re-key `params` with the model's templates, as `to_samples` does, so the model's + # leaves below match any index form in `params`; keep only the parts the model holds. + supplied = VarNamedTuple() + for (vn, value) in pairs(params) + # Must read `params` at the model's keys, not expand `value`: expanding reads + # unassigned elements and treats a `Tuple` as a single leaf. + leaves = if haskey(prediction.params, vn) + ((vn, value),) + else + model_leaves = Iterators.filter(l -> subsumes(vn, l), keys(prediction.params)) + ((l, params[l]) for l in model_leaves if haskey(params, l)) + end + for (leaf, leaf_value) in leaves + template = prediction.params.data[AbstractPPL.getsym(leaf)] + supplied = templated_setindex!!(supplied, leaf_value, leaf, template) + end + end predicted_params = VarNamedTuple() # Raw values retain sampled-variable boundaries before arrays are densified. for (vn, value) in pairs(get_raw_values(vi)) leaves = AbstractPPL.varname_and_value_leaves(vn, value) - isempty(leaves) && haskey(params, vn) && !ismissing(params[vn]) && continue + isempty(leaves) && haskey(supplied, vn) && !ismissing(supplied[vn]) && continue keep_all = all( - p -> !haskey(params, first(p)) || ismissing(params[first(p)]), leaves + p -> !haskey(supplied, first(p)) || ismissing(supplied[first(p)]), leaves ) retained = keep_all ? ((vn, value),) : leaves for (leaf, leaf_value) in retained - if keep_all || !haskey(params, leaf) || ismissing(params[leaf]) + if keep_all || !haskey(supplied, leaf) || ismissing(supplied[leaf]) predicted_params = templated_setindex!!( predicted_params, leaf_value, diff --git a/src/varnamedtuple/getset.jl b/src/varnamedtuple/getset.jl index e9f8ace62..2e0f135c2 100644 --- a/src/varnamedtuple/getset.jl +++ b/src/varnamedtuple/getset.jl @@ -52,6 +52,7 @@ function _getindex_optic( return _getindex_optic(getindex(vnt.data, S), optic.child, orig_vn) end function _getindex_optic(pa::PartialArray, optic::AbstractPPL.Index, orig_vn) + _check_index_form(pa, optic.ix, orig_vn) coptic = AbstractPPL.concretize_top_level(optic, pa.data) child_value = if _is_multiindex(pa, coptic.ix...; coptic.kw...) && @@ -77,12 +78,17 @@ end return AbstractPPL.canview(head, x) && _haskey_optic(head(x), o.child) end function _haskey_optic(vnt::VarNamedTuple, name::VarName) - return _haskey_optic(vnt, AbstractPPL.varname_to_optic(name)) + return _haskey_optic(vnt, AbstractPPL.varname_to_optic(name), name) end +# `orig_vn` only names the variable in errors, as for `_getindex_optic`. +_haskey_optic(x, optic::AbstractPPL.AbstractOptic, orig_vn) = _haskey_optic(x, optic) @inline _haskey_optic(@nospecialize(::Any), ::AbstractPPL.Iden) = true @inline _haskey_optic(::VarNamedTuple, ::AbstractPPL.Index) = false -function _haskey_optic(vnt::VarNamedTuple, optic::AbstractPPL.Property{S}) where {S} - return Base.haskey(vnt.data, S) && _haskey_optic(getindex(vnt.data, S), optic.child) +function _haskey_optic( + vnt::VarNamedTuple, optic::AbstractPPL.Property{S}, orig_vn=@varname(_) +) where {S} + return Base.haskey(vnt.data, S) && + _haskey_optic(getindex(vnt.data, S), optic.child, orig_vn) end function _haskey_optic(pa::PartialArray, ::AbstractPPL.Iden) # no masked indices @@ -96,7 +102,8 @@ function _haskey_optic(pa::PartialArray, ::AbstractPPL.Iden) return true end end -function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index) +function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index, orig_vn=@varname(_)) + _check_index_form(pa, optic.ix, orig_vn) coptic = AbstractPPL.concretize_top_level(optic, pa.data) # check the top level Index Base.haskey(pa, coptic.ix...; coptic.kw...) || return false @@ -104,9 +111,11 @@ function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index) return if optic.child isa AbstractPPL.Iden true elseif _is_multiindex(pa, coptic.ix...; coptic.kw...) - _haskey_optic(_subset_partialarray(pa, coptic.ix...; coptic.kw...), optic.child) + _haskey_optic( + _subset_partialarray(pa, coptic.ix...; coptic.kw...), optic.child, orig_vn + ) else - _haskey_optic(getindex(pa, coptic.ix...; coptic.kw...), optic.child) + _haskey_optic(getindex(pa, coptic.ix...; coptic.kw...), optic.child, orig_vn) end end function _haskey_optic(x::Tuple, optic::AbstractPPL.Index) diff --git a/src/varnamedtuple/partial_array.jl b/src/varnamedtuple/partial_array.jl index f5e2225ed..c0c398c3b 100644 --- a/src/varnamedtuple/partial_array.jl +++ b/src/varnamedtuple/partial_array.jl @@ -370,6 +370,36 @@ function _can_get_arraylikeblock(pa_data::AbstractArray) return true end +# Must throw, not report a miss, for an index that is resolved against the shape of storage +# built without a template (`begin`, `end`, `:`), a Boolean mask whose axes differ from the +# storage's, or an index with another number of dimensions: the element it names depends +# on the unknown shape. +function _check_index_form(pa::PartialArray, inds, vn) + pa.data isa GrowableArray || return nothing + n = 0 + masks_fit = true + for ind in inds + if ind isa AbstractArray{Bool} + offset = n + masks_fit &= all(d -> axes(ind, d) == axes(pa.data, offset + d), 1:ndims(ind)) + n += ndims(ind) + else + n += length(Base.index_ndims(ind)) + end + end + resolved = !_has_colon_or_dynamicindex(inds) + resolved && masks_fit && n == ndims(pa) && return nothing + throw( + ArgumentError( + "Cannot look up `$vn` in values set with $(ndims(pa))-dimensional indices " * + "without a template: the element it names depends on the array's unknown " * + "shape. Supply a template or the whole array, or pass the `VarNamedTuple` " * + "itself rather than a `Dict` of its pairs" * + (resolved ? "; or use matching indices." : "."), + ), + ) +end + """ Base.getindex(pa::PartialArray, inds::Vararg{Any}; kw...) @@ -418,14 +448,6 @@ function Base.getindex(pa::PartialArray, inds::Vararg{Any}; kw...) throw(alb_err) end elseif val isa GrowableArray - # This code path is hit for things like `vnt[@varname(x[:])]` where `x` is a PA that - # stores a GrowableArray. We warn the user that the result may be wrong. - # TODO(penelopeysm): The DynamicIndex check actually doesn't work here because if - # you index with `x[end]` for example, `inds` is already concretised outside of this - # function. - if any(ind -> ind isa AbstractPPL.DynamicIndex || ind isa Colon, inds) - _warn_growable_array_extraction() - end return unwrap_internal_array(val) else return val @@ -674,10 +696,6 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...) end function _subset_partialarray(pa::PartialArray, inds::Vararg{Any}; kw...) - if pa.data isa GrowableArray && - any(ind -> ind isa AbstractPPL.DynamicIndex || ind isa Colon, inds) - _warn_growable_array_extraction() - end new_data = view(pa.data, inds...; kw...) new_mask = view(pa.mask, inds...; kw...) return PartialArray(new_data, new_mask) diff --git a/src/varnamedtuple/vnt.jl b/src/varnamedtuple/vnt.jl index 3e48afb0a..7dd6e156d 100644 --- a/src/varnamedtuple/vnt.jl +++ b/src/varnamedtuple/vnt.jl @@ -9,6 +9,9 @@ efficient and type stable manner. It is mainly used through `getindex`, `setinde shorthand for an identity `VarName`: `vnt[:x]` is equivalent to `vnt[@varname(x)]`. Only symbols that are valid Julia identifiers are accepted. Use a `VarName` for indexed or nested variables. `Symbol("x[1]")` and `Symbol("x.y")` are rejected rather than parsed. +`haskey` and `getindex` throw an `ArgumentError` for an index whose number of dimensions +differs from that of values set by index without a template, or that uses `begin`, `end`, +`:` or a Boolean mask of a different shape on such values. Other notable methods are `merge` and `subset`. `VarNamedTuple` has an ordering to its elements, and two `VarNamedTuple`s with the same keys diff --git a/test/model.jl b/test/model.jl index 1315aa5a2..69e772cb0 100644 --- a/test/model.jl +++ b/test/model.jl @@ -704,6 +704,64 @@ const GDEMO_DEFAULT = DynamicPPL.TestUtils.demo_assume_observe_literal() end end +@model function linear_local() + y = zeros(2, 2) + y[1] ~ Normal() + return y[1] +end +@model function aliased_local() + y = zeros(3, 3) + y[4] ~ Normal() + return y[4] +end +@model function last_local() + y = zeros(3) + y[end] ~ Normal() + return y[3] +end +@model function masked_local() + y = zeros(3) + return y[[true, true, false]] ~ MvNormal(zeros(2), I) +end +@testset "values keyed in another index form without a template" begin + vi = VarInfo(RawValueAccumulator(false)) + cartesian = Dict(@varname(y[1, 1]) => 1.0) + @test_throws ArgumentError init!!( + Xoshiro(1), linear_local(), vi, InitFromParams(cartesian, InitFromPrior()) + ) + untemplated = VarNamedTuple(pairs(cartesian)) + @test_throws ArgumentError haskey(untemplated, @varname(y[1])) + @test_throws "`y[1]`" untemplated[@varname(y[1])] + @test haskey(untemplated, @varname(y[1, 1])) && !haskey(untemplated, @varname(y[2, 2])) + # Storage built from these keys is 2x2, where `y[4]` is `y[2, 2]` rather than `y[1, 2]`. + aliased = Dict(@varname(y[1, 2]) => 1.0, @varname(y[2, 2]) => 2.0) + @test_throws ArgumentError init!!( + Xoshiro(1), aliased_local(), vi, InitFromParams(aliased, nothing) + ) + linear = VarNamedTuple(pairs(Dict(@varname(y[1]) => 1.0))) + @test isempty(DynamicPPL.predict(Xoshiro(1), linear_local(), linear).params) + whole = VarNamedTuple(; y=[1.0 0.0; 0.0 0.0]) + @test isempty(DynamicPPL.predict(Xoshiro(1), linear_local(), whole).params) + # Storage built from these keys has length 2, where `end` names `y[2]`. + short = Dict(@varname(y[1]) => 1.0, @varname(y[2]) => 2.0) + @test_throws ArgumentError returned(last_local(), short) + @test_throws ArgumentError rand(Xoshiro(1), condition(last_local(), short)) + @test_throws ArgumentError logjoint(condition(last_local(), short), VarNamedTuple()) + @test_throws ArgumentError rand(Xoshiro(1), condition(masked_local(), short)) +end + +@model function first_of_two() + x = Vector{Vector{Float64}}(undef, 2) + return x[1] ~ MvNormal(zeros(2), I) +end +@testset "predict reads supplied values only where the model has variables" begin + xs = Vector{Vector{Float64}}(undef, 2) + xs[1] = [1.0, 2.0] + predicted(x) = DynamicPPL.predict(Xoshiro(1), first_of_two(), VarNamedTuple(; x)).params + @test isempty(predicted(xs)) + @test isempty(predicted((xs[1],))) +end + @info "Completed $(@__FILE__) in $(now() - __now__)." end # module diff --git a/test/varnamedtuple.jl b/test/varnamedtuple.jl index a6941b5c0..1a92b1798 100644 --- a/test/varnamedtuple.jl +++ b/test/varnamedtuple.jl @@ -965,14 +965,17 @@ end x[2] := 2.0 end warnmsg = r"Returning a `Base.Array` with" - # These should warn. There are various calls to _warn_growable_array() inside - # the VNT code to handle each of these cases. - @test_logs (:warn, warnmsg) vnt[@varname(x[:][1])] - @test_logs (:warn, warnmsg) vnt[@varname(x[:])] @test_logs (:warn, warnmsg) vnt[@varname(x)] - @test_logs (:warn, warnmsg) vnt[@varname(x[begin])] - @test_logs (:warn, warnmsg) vnt[@varname(x[end])] - @test_logs (:warn, warnmsg) vnt[@varname(x[1:end])] + # Indices resolved against the presumed size throw instead. + for vn in ( + @varname(x[:][1]), + @varname(x[:]), + @varname(x[begin]), + @varname(x[end]), + @varname(x[1:end]), + ) + @test_throws ArgumentError vnt[vn] + end # These shouldn't warn because they are safe @test_logs vnt[@varname(x[1:2])] @test_logs vnt[@varname(x[1])] @@ -1096,7 +1099,6 @@ end @test @inferred(getindex(vnt, @varname(x[1]))) == 1.0 # Check that we can get both indices @test @inferred(getindex(vnt, @varname(x[1:2]))) == [1.0, 2.0] - @test @inferred(getindex(vnt, @varname(x[:]))) == [1.0, 2.0] @test @inferred(getindex(vnt, @varname(x))) == [1.0, 2.0] @test keys(vnt) == [@varname(x[1]), @varname(x[2])]