Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions HISTORY.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,22 @@
# 0.43.0 (unreleased)

- Growable-array creation warnings use a logging limit of one per array dimension and logger ([#1538](https://github.com/TuringLang/DynamicPPL.jl/pull/1538)).
- `predict` on `SamplingOutput` accepts `multithreaded=true` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)).
- `predict` and pointwise log-density functions lost iteration labels when returning `MCMCChains.Chains` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)).
- Re-evaluating chain draws no longer shares a mutable accumulator between draws ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)).
`VarNamedTuple` records indexed random variables in same-type tuple templates ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)).

`fix` and `condition` now apply same-type tuple values to indexed variables: for `x = (0.0, 0.0); x[1] ~ Normal()`, `fix(model; x=(1.0, 2.0))` previously had no effect and now fixes `x[1]`. Mixed-type tuple values, such as `x=(1.0, 2)`, throw an `ArgumentError`, as does indexing into one stored in a `VarNamedTuple` ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)).

`VarNamedTuple` accepts Boolean-mask and empty integer-vector indices without a template: `setindex!!(vnt, [1.0, 2.0], @varname(x[[true, false, true]]))` previously threw a `BoundsError`, and `setindex!!(vnt, Float64[], @varname(x[Int[]]))` an `ArgumentError` ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)).

`VarNamedTuple` writes and merges that widen non-`Array` storage with unassigned elements, such as an `OffsetArray` of vectors, no longer throw an `UndefRefError` ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)).

`templated_setindex!!` accepts another `VarNamedTuple`'s storage as a template when a slice of it, such as `z[1:2]`, holds a `VarNamedTuple`; it previously threw ([#1539](https://github.com/TuringLang/DynamicPPL.jl/pull/1539)).

Growable-array creation warnings use a logging limit of one per array dimension and logger ([#1538](https://github.com/TuringLang/DynamicPPL.jl/pull/1538)).

`predict` on `SamplingOutput` accepts `multithreaded=true` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)).

`predict` and pointwise log-density functions lost iteration labels when returning `MCMCChains.Chains` ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)).

Re-evaluating chain draws no longer shares a mutable accumulator between draws ([#1530](https://github.com/TuringLang/DynamicPPL.jl/pull/1530)).

`predict([rng,] model, params::VarNamedTuple)` now predicts from a single draw and returns a `ParamsWithStats`; set `include_all=true` to retain supplied parameters.

Expand Down
19 changes: 9 additions & 10 deletions src/varnamedtuple.jl
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,14 @@ export VarNamedTuple,
"""
NoTemplate

A singleton struct representing the fact that there is no template for a top-level variable.
When `NoTemplate` is used, several things happen:
Represent the absence of a storage template for a top-level variable.

- If you attempt to call `make_leaf` with an Index optic, this will error.
- When recursing into substructures, `NoTemplate` will be propagated.
Low-level `VarNamedTuple` construction can create growable arrays for indexed addresses
without a template. A concrete template establishes the full extent and resolves dynamic
indices.

Collectively this means that you can only set values for variables that only have Property
optics, unless a template is provided.

It might seem more idiomatic to use `nothing` or `missing` for this. However, this causes a
bug with BangBang.setindex!!: https://github.com/JuliaFolds2/BangBang.jl/issues/43 so we use
a dedicated struct instead.
A dedicated marker avoids ambiguous dispatch in `BangBang.setindex!!` with `nothing`
or `missing`: https://github.com/JuliaFolds2/BangBang.jl/issues/43.
"""
struct NoTemplate end

Expand Down Expand Up @@ -59,6 +55,9 @@ end
SkipTemplate{N}(v::SkipTemplate{M}) where {N,M} = SkipTemplate{N + M}(v.value)
SkipTemplate{0}(v::SkipTemplate{M}) where {M} = SkipTemplate{M}(v.value)

# The array a template indexes: a PartialArray's storage, otherwise the template itself.
unwrap_template(template) = template

"""
abstract type SetPermissions end

Expand Down
95 changes: 61 additions & 34 deletions src/varnamedtuple/getset.jl
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ function _getindex_optic(vnt::VarNamedTuple, vn::VarName, orig_vn)
end

@inline _getindex_optic(@nospecialize(x::Any), ::AbstractPPL.Iden, orig_vn) = x
@inline _getindex_optic(x::Any, o::AbstractPPL.AbstractOptic, orig_vn) = o(x)
@inline function _getindex_optic(x::Any, o::AbstractPPL.AbstractOptic, orig_vn)
_reject_mixed_tuple(x)
return _getindex_optic(AbstractPPL.ohead(o)(x), o.child, orig_vn)
end
function _getindex_optic(
vnt::VarNamedTuple, optic::AbstractPPL.Property{S}, orig_vn
) where {S}
Expand Down Expand Up @@ -69,7 +72,10 @@ function _getindex_optic(arr::AbstractArray, optic::IndexWithoutChild, orig_vn)
end

# Fallback for everything else.
@inline _haskey_optic(x::Any, o::AbstractPPL.AbstractOptic) = AbstractPPL.canview(o, x)
@inline function _haskey_optic(x::Any, o::AbstractPPL.AbstractOptic)
head = AbstractPPL.ohead(o)
return AbstractPPL.canview(head, x) && _haskey_optic(head(x), o.child)
end
function _haskey_optic(vnt::VarNamedTuple, name::VarName)
return _haskey_optic(vnt, AbstractPPL.varname_to_optic(name))
end
Expand Down Expand Up @@ -103,6 +109,12 @@ function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index)
_haskey_optic(getindex(pa, coptic.ix...; coptic.kw...), optic.child)
end
end
function _haskey_optic(x::Tuple, optic::AbstractPPL.Index)
coptic = AbstractPPL.concretize_top_level(optic, x)
isempty(coptic.kw) && length(coptic.ix) == 1 || return false
checkbounds(Bool, keys(x), coptic.ix...) || return false
return _haskey_optic(getindex(x, coptic.ix...; coptic.kw...), coptic.child)
end
function _haskey_optic(arr::AbstractArray, optic::IndexWithoutChild)
# Note that this call to `checkbounds` can error, although it is technically out of our
# hands: it depends on how the provider of the AbstractArray has implemented
Expand Down Expand Up @@ -171,16 +183,35 @@ end
# `x.a ~ dist` would error if `x` did not have a property `a`.
(::SharedGetProperty{S})(x) where {S} = getproperty(x, S)

# Template for the entry that optic's index selects; NoTemplate when it cannot be indexed.
index_template(template::SkipTemplate, optic) = decrease_skip(template)
index_template(template, optic) = NoTemplate()
function index_template(template::Union{Tuple,AbstractArray}, optic)
return getindex(template, optic.ix...; optic.kw...)
end
unwrap_template(template::PartialArray) = template.data
function index_template(template::PartialArray, optic)
# Read through the PartialArray so a slice namespace yields its block, rather
# than an array of ArrayLikeBlocks, before descending into the child's fields.
return if (eltype(template) <: ArrayLikeBlock || ArrayLikeBlock <: eltype(template)) &&
haskey(template, optic.ix...; optic.kw...)
getindex(template, optic.ix...; optic.kw...)
else
index_template(template.data, optic)
end
end

function _setindex_optic!!(
pa::PartialArray, value, optic::AbstractPPL.Index, template, permissions::SetPermissions
)
_reject_mixed_tuple(template)
need_merge = false
coptic = AbstractPPL.concretize_top_level(optic, pa.data)
# We might be attempting to set a slice into a PartialArray{GrowableArray} that
# doesn't yet have enough indices for that slice. Expand it if so.
pa = grow_to_indices!!(pa, coptic.ix...; coptic.kw...)

is_multiindex = _is_multiindex(template, coptic.ix...; coptic.kw...)
is_multiindex = _is_multiindex(pa.data, coptic.ix...; coptic.kw...)

if permissions isa MustNotOverwrite && optic.child isa AbstractPPL.Iden
if any(view(pa.mask, coptic.ix...; coptic.kw...))
Expand All @@ -192,19 +223,14 @@ function _setindex_optic!!(
# Skip recursion
value
else
child_template = if template === NoTemplate()
NoTemplate()
elseif template isa SkipTemplate
decrease_skip(template)
elseif is_multiindex
Base.getindex(template, coptic.ix...; coptic.kw...)
elseif isassigned(template, coptic.ix...; coptic.kw...)
# Single-index, but we should check that there is actual data there before
# calling getindex, as otherwise it will error
Base.getindex(template, coptic.ix...; coptic.kw...)
else
NoTemplate()
end
child_template =
if template isa Union{AbstractArray,PartialArray} &&
!is_multiindex &&
!isassigned(template, coptic.ix...; coptic.kw...)
NoTemplate()
else
index_template(template, coptic)
end

# TODO(penelopeysm): This check to haskey() will check *all* the indices, it only
# returns true if they are all filled. This is probably not really correct, and is
Expand Down Expand Up @@ -337,25 +363,34 @@ end
# This function handles Index optics. Since Index optics can represent either single-element
# indexing or multi-element indexing (e.g., slices, arrays of indices), and their
# implementation can be somewhat different, we dispatch to separate functions for each case.
function make_leaf(value, optic::AbstractPPL.Index, template::PartialArray)
# If the template is a PA, use its data as the template.
return make_leaf(value, optic, template.data)
end
function make_leaf(
value,
optic::AbstractPPL.Index,
template::Union{AbstractArray,NoTemplate,SkipTemplate,Missing},
template::Union{AbstractArray,PartialArray,NoTemplate,SkipTemplate,Missing},
)
# First we need to resolve any dynamic indices, since _is_multiindex doesn't work with
# them. This also helpfully catches errors if there is a dynamic index and a suitable
# template is not provided (e.g., if someone tries to set `x[end]` without a template).
coptic = AbstractPPL.concretize_top_level(optic, template)
return if _is_multiindex(template, coptic.ix...; coptic.kw...)
coptic = AbstractPPL.concretize_top_level(optic, unwrap_template(template))
return if _is_multiindex(unwrap_template(template), coptic.ix...; coptic.kw...)
make_leaf_multiindex(value, coptic, template)
else
make_leaf_singleindex(value, coptic, template)
end
end
# Tuple nodes are stored as arrays (collect), which cannot keep mixed element types.
_reject_mixed_tuple(x) = nothing
_reject_mixed_tuple(::Tuple{T,Vararg{T}}) where {T} = nothing
_reject_mixed_tuple(::Tuple{}) = nothing
function _reject_mixed_tuple(::Tuple)
throw(
ArgumentError("same-type tuples only; give the elements one type or use an array")
)
end
function make_leaf(value, optic::AbstractPPL.Index, template::Tuple)
_reject_mixed_tuple(template)
return make_leaf(value, optic, collect(template))
end
function make_leaf(value, ::AbstractPPL.Index, template::Any)
# Fallback method; this is hit for other indexable things that we don't know how to deal
# with.
Expand All @@ -368,17 +403,7 @@ function make_sub_value(value, coptic::AbstractPPL.Index, template)
return if coptic.child isa AbstractPPL.Iden
value
else
child_template = if template isa NoTemplate
NoTemplate()
elseif template isa SkipTemplate
decrease_skip(template)
elseif template isa AbstractArray
# Note template can't be a PartialArray as that would have been unwrapped in
# another make_leaf method.
getindex(template, coptic.ix...; coptic.kw...)
else
NoTemplate()
end
child_template = index_template(template, coptic)
make_leaf(value, coptic.child, child_template)
end
end
Expand All @@ -388,6 +413,7 @@ end
# by the caller.
function make_leaf_singleindex(value, coptic::AbstractPPL.Index, template)
sub_value = make_sub_value(value, coptic, template)
template = unwrap_template(template)
# `sub_value` will become a single element inside the PartialArray that we create. To
# ensure type stability, we want to make sure that the PartialArray is created with the
# correct eltype to begin with, otherwise setindex!! may become type unstable.
Expand Down Expand Up @@ -418,6 +444,7 @@ end
# This is more complex.
function make_leaf_multiindex(value, coptic::AbstractPPL.Index, template)
sub_value = make_sub_value(value, coptic, template)
template = unwrap_template(template)

# Firstly, we need to make sure that sub_value has *exactly* the right size to fit into
# the indices specified by `coptic`. This might not always be the case. Consider
Expand Down
60 changes: 49 additions & 11 deletions src/varnamedtuple/partial_array.jl
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,8 @@ end
# Helper functions to determine the largest index from various index types.
largest_index(ix::Integer) = ix
largest_index(r::AbstractUnitRange) = last(r)
largest_index(r::AbstractVector{<:Integer}) = maximum(r)
largest_index(r::AbstractVector{<:Integer}) = isempty(r) ? zero(eltype(r)) : maximum(r)
largest_index(r::AbstractVector{Bool}) = length(r)
function largest_index(x)
throw(
ArgumentError(
Expand Down Expand Up @@ -204,6 +205,33 @@ Base.eltype(::PartialArray{ElType}) where {ElType} = ElType
Base.size(pa::PartialArray) = size(pa.data)
Base.isassigned(pa::PartialArray, ix...; kw...) = isassigned(pa.data, ix...; kw...)

# Copy only assigned entries: reading an unassigned non-isbits element throws UndefRefError.
function _copy_partial_array_data(data, mask, ::Type{T}) where {T}
result = similar(data, T)
@inbounds for i in eachindex(mask)
mask[i] && (result[i] = data[i])
end
return result
end

# setindex!! for PartialArray storage that never reads unassigned non-isbits entries.
function _setindex_partial_array_data!!(data, mask, value, inds...; kw...)
if !isbitstype(eltype(data))
value_type = if value isa AbstractArray && _is_multiindex(data, inds...; kw...)
eltype(value)
else
typeof(value)
end
new_eltype = promote_type(eltype(data), value_type)
if !(new_eltype <: eltype(data)) || !BangBang.implements(setindex!, typeof(data))
data = _copy_partial_array_data(data, mask, new_eltype)
end
setindex!(data, value, inds...; kw...)
return data
end
return BangBang.setindex!!(data, value, inds...; kw...)
end

function Base.copy(pa::PartialArray)
# Make a shallow copy of pa, except for any VarNamedTuple elements, which we recursively
# copy.
Expand Down Expand Up @@ -467,6 +495,12 @@ function _remove_partial_blocks!!(
return pa_data, pa_mask
end

@inline function _ndims(f::Array, ix::Vararg{Union{Integer,CartesianIndex}}; kw...)
isempty(kw) || return ndims(view(f, ix...; kw...))
indices = Base.to_indices(f, ix)
@boundscheck checkbounds(f, indices...)
return 0
end
@inline function _ndims(f::AbstractArray, ix...; kw...)
return ndims(view(f, ix...; kw...))
end
Expand All @@ -480,7 +514,7 @@ end
@generated function _ndims_static(::T) where {T<:Tuple}
i = 0
for x in T.parameters
if x <: AbstractVector{<:Int} || x <: Colon
if x <: AbstractVector{<:Integer} || x <: Colon
i += 1
end
end
Expand All @@ -495,7 +529,7 @@ end
end
@generated function _is_multiindex_static(::T) where {T<:Tuple}
for x in T.parameters
if x <: AbstractVector{<:Int} || x <: Colon
if x <: AbstractVector{<:Integer} || x <: Colon
return :(return true)
end
end
Expand Down Expand Up @@ -580,7 +614,9 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...)
if _needs_arraylikeblock(new_data, value, inds...; kw...)
idx_sz = size(@view new_data[inds..., kw...])
alb = ArrayLikeBlock(value, inds, NamedTuple(kw), idx_sz)
new_data = setindex!!(new_data, fill(alb, idx_sz...), inds...; kw...)
new_data = _setindex_partial_array_data!!(
new_data, new_mask, fill(alb, idx_sz...), inds...; kw...
)
fill!(view(new_mask, inds...; kw...), true)
else
if value isa PartialArray
Expand All @@ -607,9 +643,7 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...)
new_data = if new_eltype <: eltype(new_data)
new_data
else
broadened = similar(new_data, new_eltype)
copy!(broadened, new_data)
broadened
_copy_partial_array_data(new_data, new_mask, new_eltype)
end
new_data_view = view(new_data, inds...; kw...)
new_mask_view = view(new_mask, inds...; kw...)
Expand All @@ -624,11 +658,15 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...)
else
# Overwriting one element of a PA with another PA. The PA is the value
# itself! -- i.e. nested PAs! This can happen with things like x[1][1]
new_data = setindex!!(new_data, value, inds...; kw...)
new_data = _setindex_partial_array_data!!(
new_data, new_mask, value, inds...; kw...
)
setindex!(new_mask, true, inds...; kw...)
end
else
new_data = setindex!!(new_data, value, inds...; kw...)
new_data = _setindex_partial_array_data!!(
new_data, new_mask, value, inds...; kw...
)
fill!(view(new_mask, inds...; kw...), true)
end
end
Expand Down Expand Up @@ -707,10 +745,10 @@ function _merge(pa1::PartialArray, pa2::PartialArray, recurse::Val)
for i in eachindex(pa1.mask)
if pa1.mask[i]
new_elem, new_mask_val = _merge_element(pa1, pa2, i, recurse)
new_mask[i] = new_mask_val
if new_mask_val
new_data = setindex!!(new_data, new_elem, i)
new_data = _setindex_partial_array_data!!(new_data, new_mask, new_elem, i)
end
new_mask[i] = new_mask_val
end
end
return _concretise_eltype!!(PartialArray(new_data, new_mask))
Expand Down
20 changes: 20 additions & 0 deletions test/compiler.jl
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,26 @@ end
end
x = vdemo()()
@test all((isassigned(x, i) for i in eachindex(x)))

@testset "tuple Index nodes" begin
@model function tuple_local()
x = (0.0, 0.0)
x[1] ~ Normal()
x[2] ~ Normal()
return x
end
draw = rand(Random.Xoshiro(1), tuple_local())
@test collect(keys(draw)) == [@varname(x[1]), @varname(x[2])]
@test [draw[@varname(x[1])], draw[@varname(x[2])]] ==
rand(Random.Xoshiro(1), Normal(), 2)
@test fix(tuple_local(); x=(1.0, 2.0))(Random.Xoshiro(1)) == (1.0, 2.0)
restriction = ArgumentError(
"same-type tuples only; give the elements one type or use an array"
)
@test_throws restriction fix(tuple_local(); x=(1.0f0, big"2.0"))(
Random.Xoshiro(1)
)
end
end

# Test that that using @. to stop unwanted broadcasting on the RHS works.
Expand Down
Loading
Loading