diff --git a/HISTORY.md b/HISTORY.md index cbaa0a31d..ceeaaf6dc 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,9 +1,22 @@ # 0.43.0 (unreleased) - - Growable-array creation warnings use a logging limit of one per array dimension and logger ([#1538](https://github.com/TuringLang/DynamicPPL.jl/pull/1538)). - - `predict` on `SamplingOutput` accepts `multithreaded=true` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)). - - `predict` and pointwise log-density functions lost iteration labels when returning `MCMCChains.Chains` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)). - - Re-evaluating chain draws no longer shares a mutable accumulator between draws ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)). +`VarNamedTuple` records indexed random variables in same-type tuple templates ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)). + +`fix` and `condition` now apply same-type tuple values to indexed variables: for `x = (0.0, 0.0); x[1] ~ Normal()`, `fix(model; x=(1.0, 2.0))` previously had no effect and now fixes `x[1]`. Mixed-type tuple values, such as `x=(1.0, 2)`, throw an `ArgumentError`, as does indexing into one stored in a `VarNamedTuple` ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)). + +`VarNamedTuple` accepts Boolean-mask and empty integer-vector indices without a template: `setindex!!(vnt, [1.0, 2.0], @varname(x[[true, false, true]]))` previously threw a `BoundsError`, and `setindex!!(vnt, Float64[], @varname(x[Int[]]))` an `ArgumentError` ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)). + +`VarNamedTuple` writes and merges that widen non-`Array` storage with unassigned elements, such as an `OffsetArray` of vectors, no longer throw an `UndefRefError` ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)). + +`templated_setindex!!` accepts another `VarNamedTuple`'s storage as a template when a slice of it, such as `z[1:2]`, holds a `VarNamedTuple`; it previously threw ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)). + +Growable-array creation warnings use a logging limit of one per array dimension and logger ([#1538](https://github.com/TuringLang/DynamicPPL.jl/pull/1538)). + +`predict` on `SamplingOutput` accepts `multithreaded=true` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)). + +`predict` and pointwise log-density functions lost iteration labels when returning `MCMCChains.Chains` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)). + +Re-evaluating chain draws no longer shares a mutable accumulator between draws ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)). `predict([rng,] model, params::VarNamedTuple)` now predicts from a single draw and returns a `ParamsWithStats`; set `include_all=true` to retain supplied parameters. diff --git a/src/varnamedtuple.jl b/src/varnamedtuple.jl index 8c087ff4f..c53cf004d 100644 --- a/src/varnamedtuple.jl +++ b/src/varnamedtuple.jl @@ -20,18 +20,14 @@ export VarNamedTuple, """ NoTemplate -A singleton struct representing the fact that there is no template for a top-level variable. -When `NoTemplate` is used, several things happen: +Represent the absence of a storage template for a top-level variable. - - If you attempt to call `make_leaf` with an Index optic, this will error. - - When recursing into substructures, `NoTemplate` will be propagated. +Low-level `VarNamedTuple` construction can create growable arrays for indexed addresses +without a template. A concrete template establishes the full extent and resolves dynamic +indices. -Collectively this means that you can only set values for variables that only have Property -optics, unless a template is provided. - -It might seem more idiomatic to use `nothing` or `missing` for this. However, this causes a -bug with BangBang.setindex!!: https://github.com/JuliaFolds2/BangBang.jl/issues/43 so we use -a dedicated struct instead. +A dedicated marker avoids ambiguous dispatch in `BangBang.setindex!!` with `nothing` +or `missing`: https://github.com/JuliaFolds2/BangBang.jl/issues/43. """ struct NoTemplate end @@ -59,6 +55,9 @@ end SkipTemplate{N}(v::SkipTemplate{M}) where {N,M} = SkipTemplate{N + M}(v.value) SkipTemplate{0}(v::SkipTemplate{M}) where {M} = SkipTemplate{M}(v.value) +# The array a template indexes: a PartialArray's storage, otherwise the template itself. +unwrap_template(template) = template + """ abstract type SetPermissions end diff --git a/src/varnamedtuple/getset.jl b/src/varnamedtuple/getset.jl index d4593d0c6..e9f8ace62 100644 --- a/src/varnamedtuple/getset.jl +++ b/src/varnamedtuple/getset.jl @@ -39,7 +39,10 @@ function _getindex_optic(vnt::VarNamedTuple, vn::VarName, orig_vn) end @inline _getindex_optic(@nospecialize(x::Any), ::AbstractPPL.Iden, orig_vn) = x -@inline _getindex_optic(x::Any, o::AbstractPPL.AbstractOptic, orig_vn) = o(x) +@inline function _getindex_optic(x::Any, o::AbstractPPL.AbstractOptic, orig_vn) + _reject_mixed_tuple(x) + return _getindex_optic(AbstractPPL.ohead(o)(x), o.child, orig_vn) +end function _getindex_optic( vnt::VarNamedTuple, optic::AbstractPPL.Property{S}, orig_vn ) where {S} @@ -69,7 +72,10 @@ function _getindex_optic(arr::AbstractArray, optic::IndexWithoutChild, orig_vn) end # Fallback for everything else. -@inline _haskey_optic(x::Any, o::AbstractPPL.AbstractOptic) = AbstractPPL.canview(o, x) +@inline function _haskey_optic(x::Any, o::AbstractPPL.AbstractOptic) + head = AbstractPPL.ohead(o) + return AbstractPPL.canview(head, x) && _haskey_optic(head(x), o.child) +end function _haskey_optic(vnt::VarNamedTuple, name::VarName) return _haskey_optic(vnt, AbstractPPL.varname_to_optic(name)) end @@ -103,6 +109,12 @@ function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index) _haskey_optic(getindex(pa, coptic.ix...; coptic.kw...), optic.child) end end +function _haskey_optic(x::Tuple, optic::AbstractPPL.Index) + coptic = AbstractPPL.concretize_top_level(optic, x) + isempty(coptic.kw) && length(coptic.ix) == 1 || return false + checkbounds(Bool, keys(x), coptic.ix...) || return false + return _haskey_optic(getindex(x, coptic.ix...; coptic.kw...), coptic.child) +end function _haskey_optic(arr::AbstractArray, optic::IndexWithoutChild) # Note that this call to `checkbounds` can error, although it is technically out of our # hands: it depends on how the provider of the AbstractArray has implemented @@ -171,16 +183,35 @@ end # `x.a ~ dist` would error if `x` did not have a property `a`. (::SharedGetProperty{S})(x) where {S} = getproperty(x, S) +# Template for the entry that optic's index selects; NoTemplate when it cannot be indexed. +index_template(template::SkipTemplate, optic) = decrease_skip(template) +index_template(template, optic) = NoTemplate() +function index_template(template::Union{Tuple,AbstractArray}, optic) + return getindex(template, optic.ix...; optic.kw...) +end +unwrap_template(template::PartialArray) = template.data +function index_template(template::PartialArray, optic) + # Read through the PartialArray so a slice namespace yields its block, rather + # than an array of ArrayLikeBlocks, before descending into the child's fields. + return if (eltype(template) <: ArrayLikeBlock || ArrayLikeBlock <: eltype(template)) && + haskey(template, optic.ix...; optic.kw...) + getindex(template, optic.ix...; optic.kw...) + else + index_template(template.data, optic) + end +end + function _setindex_optic!!( pa::PartialArray, value, optic::AbstractPPL.Index, template, permissions::SetPermissions ) + _reject_mixed_tuple(template) need_merge = false coptic = AbstractPPL.concretize_top_level(optic, pa.data) # We might be attempting to set a slice into a PartialArray{GrowableArray} that # doesn't yet have enough indices for that slice. Expand it if so. pa = grow_to_indices!!(pa, coptic.ix...; coptic.kw...) - is_multiindex = _is_multiindex(template, coptic.ix...; coptic.kw...) + is_multiindex = _is_multiindex(pa.data, coptic.ix...; coptic.kw...) if permissions isa MustNotOverwrite && optic.child isa AbstractPPL.Iden if any(view(pa.mask, coptic.ix...; coptic.kw...)) @@ -192,19 +223,14 @@ function _setindex_optic!!( # Skip recursion value else - child_template = if template === NoTemplate() - NoTemplate() - elseif template isa SkipTemplate - decrease_skip(template) - elseif is_multiindex - Base.getindex(template, coptic.ix...; coptic.kw...) - elseif isassigned(template, coptic.ix...; coptic.kw...) - # Single-index, but we should check that there is actual data there before - # calling getindex, as otherwise it will error - Base.getindex(template, coptic.ix...; coptic.kw...) - else - NoTemplate() - end + child_template = + if template isa Union{AbstractArray,PartialArray} && + !is_multiindex && + !isassigned(template, coptic.ix...; coptic.kw...) + NoTemplate() + else + index_template(template, coptic) + end # TODO(penelopeysm): This check to haskey() will check *all* the indices, it only # returns true if they are all filled. This is probably not really correct, and is @@ -337,25 +363,34 @@ end # This function handles Index optics. Since Index optics can represent either single-element # indexing or multi-element indexing (e.g., slices, arrays of indices), and their # implementation can be somewhat different, we dispatch to separate functions for each case. -function make_leaf(value, optic::AbstractPPL.Index, template::PartialArray) - # If the template is a PA, use its data as the template. - return make_leaf(value, optic, template.data) -end function make_leaf( value, optic::AbstractPPL.Index, - template::Union{AbstractArray,NoTemplate,SkipTemplate,Missing}, + template::Union{AbstractArray,PartialArray,NoTemplate,SkipTemplate,Missing}, ) # First we need to resolve any dynamic indices, since _is_multiindex doesn't work with # them. This also helpfully catches errors if there is a dynamic index and a suitable # template is not provided (e.g., if someone tries to set `x[end]` without a template). - coptic = AbstractPPL.concretize_top_level(optic, template) - return if _is_multiindex(template, coptic.ix...; coptic.kw...) + coptic = AbstractPPL.concretize_top_level(optic, unwrap_template(template)) + return if _is_multiindex(unwrap_template(template), coptic.ix...; coptic.kw...) make_leaf_multiindex(value, coptic, template) else make_leaf_singleindex(value, coptic, template) end end +# Tuple nodes are stored as arrays (collect), which cannot keep mixed element types. +_reject_mixed_tuple(x) = nothing +_reject_mixed_tuple(::Tuple{T,Vararg{T}}) where {T} = nothing +_reject_mixed_tuple(::Tuple{}) = nothing +function _reject_mixed_tuple(::Tuple) + throw( + ArgumentError("same-type tuples only; give the elements one type or use an array") + ) +end +function make_leaf(value, optic::AbstractPPL.Index, template::Tuple) + _reject_mixed_tuple(template) + return make_leaf(value, optic, collect(template)) +end function make_leaf(value, ::AbstractPPL.Index, template::Any) # Fallback method; this is hit for other indexable things that we don't know how to deal # with. @@ -368,17 +403,7 @@ function make_sub_value(value, coptic::AbstractPPL.Index, template) return if coptic.child isa AbstractPPL.Iden value else - child_template = if template isa NoTemplate - NoTemplate() - elseif template isa SkipTemplate - decrease_skip(template) - elseif template isa AbstractArray - # Note template can't be a PartialArray as that would have been unwrapped in - # another make_leaf method. - getindex(template, coptic.ix...; coptic.kw...) - else - NoTemplate() - end + child_template = index_template(template, coptic) make_leaf(value, coptic.child, child_template) end end @@ -388,6 +413,7 @@ end # by the caller. function make_leaf_singleindex(value, coptic::AbstractPPL.Index, template) sub_value = make_sub_value(value, coptic, template) + template = unwrap_template(template) # `sub_value` will become a single element inside the PartialArray that we create. To # ensure type stability, we want to make sure that the PartialArray is created with the # correct eltype to begin with, otherwise setindex!! may become type unstable. @@ -418,6 +444,7 @@ end # This is more complex. function make_leaf_multiindex(value, coptic::AbstractPPL.Index, template) sub_value = make_sub_value(value, coptic, template) + template = unwrap_template(template) # Firstly, we need to make sure that sub_value has *exactly* the right size to fit into # the indices specified by `coptic`. This might not always be the case. Consider diff --git a/src/varnamedtuple/partial_array.jl b/src/varnamedtuple/partial_array.jl index acdaaa2ef..d31a25697 100644 --- a/src/varnamedtuple/partial_array.jl +++ b/src/varnamedtuple/partial_array.jl @@ -109,7 +109,8 @@ end # Helper functions to determine the largest index from various index types. largest_index(ix::Integer) = ix largest_index(r::AbstractUnitRange) = last(r) -largest_index(r::AbstractVector{<:Integer}) = maximum(r) +largest_index(r::AbstractVector{<:Integer}) = isempty(r) ? zero(eltype(r)) : maximum(r) +largest_index(r::AbstractVector{Bool}) = length(r) function largest_index(x) throw( ArgumentError( @@ -204,6 +205,33 @@ Base.eltype(::PartialArray{ElType}) where {ElType} = ElType Base.size(pa::PartialArray) = size(pa.data) Base.isassigned(pa::PartialArray, ix...; kw...) = isassigned(pa.data, ix...; kw...) +# Copy only assigned entries: reading an unassigned non-isbits element throws UndefRefError. +function _copy_partial_array_data(data, mask, ::Type{T}) where {T} + result = similar(data, T) + @inbounds for i in eachindex(mask) + mask[i] && (result[i] = data[i]) + end + return result +end + +# setindex!! for PartialArray storage that never reads unassigned non-isbits entries. +function _setindex_partial_array_data!!(data, mask, value, inds...; kw...) + if !isbitstype(eltype(data)) + value_type = if value isa AbstractArray && _is_multiindex(data, inds...; kw...) + eltype(value) + else + typeof(value) + end + new_eltype = promote_type(eltype(data), value_type) + if !(new_eltype <: eltype(data)) || !BangBang.implements(setindex!, typeof(data)) + data = _copy_partial_array_data(data, mask, new_eltype) + end + setindex!(data, value, inds...; kw...) + return data + end + return BangBang.setindex!!(data, value, inds...; kw...) +end + function Base.copy(pa::PartialArray) # Make a shallow copy of pa, except for any VarNamedTuple elements, which we recursively # copy. @@ -467,6 +495,12 @@ function _remove_partial_blocks!!( return pa_data, pa_mask end +@inline function _ndims(f::Array, ix::Vararg{Union{Integer,CartesianIndex}}; kw...) + isempty(kw) || return ndims(view(f, ix...; kw...)) + indices = Base.to_indices(f, ix) + @boundscheck checkbounds(f, indices...) + return 0 +end @inline function _ndims(f::AbstractArray, ix...; kw...) return ndims(view(f, ix...; kw...)) end @@ -480,7 +514,7 @@ end @generated function _ndims_static(::T) where {T<:Tuple} i = 0 for x in T.parameters - if x <: AbstractVector{<:Int} || x <: Colon + if x <: AbstractVector{<:Integer} || x <: Colon i += 1 end end @@ -495,7 +529,7 @@ end end @generated function _is_multiindex_static(::T) where {T<:Tuple} for x in T.parameters - if x <: AbstractVector{<:Int} || x <: Colon + if x <: AbstractVector{<:Integer} || x <: Colon return :(return true) end end @@ -580,7 +614,9 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...) if _needs_arraylikeblock(new_data, value, inds...; kw...) idx_sz = size(@view new_data[inds..., kw...]) alb = ArrayLikeBlock(value, inds, NamedTuple(kw), idx_sz) - new_data = setindex!!(new_data, fill(alb, idx_sz...), inds...; kw...) + new_data = _setindex_partial_array_data!!( + new_data, new_mask, fill(alb, idx_sz...), inds...; kw... + ) fill!(view(new_mask, inds...; kw...), true) else if value isa PartialArray @@ -607,9 +643,7 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...) new_data = if new_eltype <: eltype(new_data) new_data else - broadened = similar(new_data, new_eltype) - copy!(broadened, new_data) - broadened + _copy_partial_array_data(new_data, new_mask, new_eltype) end new_data_view = view(new_data, inds...; kw...) new_mask_view = view(new_mask, inds...; kw...) @@ -624,11 +658,15 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...) else # Overwriting one element of a PA with another PA. The PA is the value # itself! -- i.e. nested PAs! This can happen with things like x[1][1] - new_data = setindex!!(new_data, value, inds...; kw...) + new_data = _setindex_partial_array_data!!( + new_data, new_mask, value, inds...; kw... + ) setindex!(new_mask, true, inds...; kw...) end else - new_data = setindex!!(new_data, value, inds...; kw...) + new_data = _setindex_partial_array_data!!( + new_data, new_mask, value, inds...; kw... + ) fill!(view(new_mask, inds...; kw...), true) end end @@ -707,10 +745,10 @@ function _merge(pa1::PartialArray, pa2::PartialArray, recurse::Val) for i in eachindex(pa1.mask) if pa1.mask[i] new_elem, new_mask_val = _merge_element(pa1, pa2, i, recurse) - new_mask[i] = new_mask_val if new_mask_val - new_data = setindex!!(new_data, new_elem, i) + new_data = _setindex_partial_array_data!!(new_data, new_mask, new_elem, i) end + new_mask[i] = new_mask_val end end return _concretise_eltype!!(PartialArray(new_data, new_mask)) diff --git a/test/compiler.jl b/test/compiler.jl index 42cab20c7..9bc2036ce 100644 --- a/test/compiler.jl +++ b/test/compiler.jl @@ -340,6 +340,26 @@ end end x = vdemo()() @test all((isassigned(x, i) for i in eachindex(x))) + + @testset "tuple Index nodes" begin + @model function tuple_local() + x = (0.0, 0.0) + x[1] ~ Normal() + x[2] ~ Normal() + return x + end + draw = rand(Random.Xoshiro(1), tuple_local()) + @test collect(keys(draw)) == [@varname(x[1]), @varname(x[2])] + @test [draw[@varname(x[1])], draw[@varname(x[2])]] == + rand(Random.Xoshiro(1), Normal(), 2) + @test fix(tuple_local(); x=(1.0, 2.0))(Random.Xoshiro(1)) == (1.0, 2.0) + restriction = ArgumentError( + "same-type tuples only; give the elements one type or use an array" + ) + @test_throws restriction fix(tuple_local(); x=(1.0f0, big"2.0"))( + Random.Xoshiro(1) + ) + end end # Test that that using @. to stop unwanted broadcasting on the RHS works. diff --git a/test/varnamedtuple.jl b/test/varnamedtuple.jl index bda27ba79..456700201 100644 --- a/test/varnamedtuple.jl +++ b/test/varnamedtuple.jl @@ -871,6 +871,45 @@ end @test @inferred(vnt[@varname(x[1][1, 2])]) == 2.0 end + @testset "Boolean-mask and empty indices" begin + mask = [true, false, true] + vnt = setindex!!(VarNamedTuple(), [1.0, 2.0], @varname(x[mask])) + @test keys(vnt) == [@varname(x[1]), @varname(x[3])] + @test vnt[@varname(x[mask])] == [1.0, 2.0] + vnt = setindex!!(VarNamedTuple(), Float64[], @varname(x[Int[]])) + @test vnt[@varname(x[Int[]])] == Float64[] + end + + @testset "widening storage with unassigned slots" begin + template = OA.OffsetArray(Vector{Vector{Float64}}(undef, 3), 0:2) + vector = templated_setindex!!(VarNamedTuple(), [1.0], @varname(x[1]), template) + str = templated_setindex!!(VarNamedTuple(), "a", @varname(x[0]), template) + @test merge(str, vector) == + templated_setindex!!(copy(vector), "a", @varname(x[0]), template) + promoted = templated_setindex!!(vector, [2], @varname(x[2]), template) + @test promoted[@varname(x[2])] == [2.0] + end + + @testset "unassigned template entry" begin + array = Vector{Vector{Float64}}(undef, 2) + partial = templated_setindex!!(VarNamedTuple(), [2.0], @varname(x[2]), array) + for template in (array, partial.data.x) + vnt = templated_setindex!!(VarNamedTuple(), [2.0], @varname(x[2]), template) + vnt = templated_setindex!!(vnt, 1.0, @varname(x[1][1]), template) + @test vnt[@varname(x[1][1])] == 1.0 + end + end + + @testset "PartialArray template holding a slice block" begin + slices = fill(VarNamedTuple(), 3) + inner = VarNamedTuple(; a=[1.0, 1.0]) + template = templated_setindex!!( + VarNamedTuple(), inner, @varname(z[1:2]), slices + ) + vn = @varname(z[1:2].a[1]) + @test templated_setindex!!(VarNamedTuple(), 3.0, vn, template.data.z)[vn] == 3.0 + end + @testset "Merging" begin # Note that there are a couple more merge tests with ALBs under the merge # section. @@ -2467,6 +2506,52 @@ end v13s = VarNamedTuple(; x=CA.ComponentArray(; a=nothing, b=nothing)) test_skeleton(v13, v13s) end + + @testset "tuple Index nodes" begin + for (template, names, values) in ( + ((0.0, 0.0), (@varname(x[1]), @varname(x[2])), (1.0, 2.0)), + ((a=(0.0, 0.0),), (@varname(x.a[1]), @varname(x.a[2])), (1.0, 2.0)), + (([0.0, 0.0], [0.0]), (@varname(x[1][2]), @varname(x[2][end])), (1.0, 2.0)), + ) + vnt = VarNamedTuple() + for (vn, value) in zip(names, values) + vnt = templated_setindex!!(vnt, value, vn, template) + @test haskey(vnt, vn) + @test vnt[vn] == value + end + end + for (value, vn) in ( + ((a=(1.0, 2.0),), @varname(x.a[2])), + ([(1.0, 2.0)], @varname(x[1][2])), + (((1.0, 2.0),), @varname(x[1][2])), + ) + whole = VarNamedTuple(; x=value) + @test haskey(whole, vn) + @test whole[vn] == 2.0 + end + restriction = ArgumentError( + "same-type tuples only; give the elements one type or use an array" + ) + for (template, vn) in ( + ((1.0f0, big"2.0"), @varname(x[1])), + ((a=(1.0f0, big"2.0"),), @varname(x.a[1])), + ([(1.0f0, big"2.0")], @varname(x[1][1])), + ) + @test_throws restriction templated_setindex!!( + VarNamedTuple(), 1.0f0, vn, template + ) + @test_throws restriction VarNamedTuple(; x=template)[vn] + end + existing = templated_setindex!!(VarNamedTuple(), 1.0f0, @varname(x[1]), zeros(2)) + @test_throws restriction templated_setindex!!( + existing, 1.0f0, @varname(x[2]), (1.0f0, big"2.0") + ) + whole = VarNamedTuple(; x=(1.0, 2.0)) + @test !haskey(whole, @varname(x[3])) + @test haskey(whole, @varname(x[:])) + @test !haskey(whole, @varname(x[1:3])) + @test VarNamedTuple(; x=())[@varname(x[:])] == () + end end @info "Completed $(@__FILE__) in $(now() - __now__)."