Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion HISTORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ Fixed model argument evaluation world-age errors when loading without a valid pr

Thread-safe tilde assignments now throw an `ArgumentError` if any container is replaced, including same-type immutable values: `(a = zero(T),)` → `Vector{T}(undef, n)`. Allocate mutable storage with `::Type{T}=Float64`; use `Vector{Union{Missing,Real}}` for imputation under AD. See [#1521](https://github.com/TuringLang/DynamicPPL.jl/pull/1521).

`VarNamedTuple` membership resolves `begin` and `end` against the stored array. See [#1490](https://github.com/TuringLang/DynamicPPL.jl/pull/1490).
`VarNamedTuple` membership resolves `begin` and `end` against the stored array when it has a template. See [#1490](https://github.com/TuringLang/DynamicPPL.jl/pull/1490).

`ComponentVector` properties, including nested fields and slices, use consistent indices for membership, retrieval, and updates. See [#1491](https://github.com/TuringLang/DynamicPPL.jl/pull/1491).

Expand All @@ -46,6 +46,8 @@ Added `evaluate!!(model, context, vi)` to evaluate with an explicit leaf context

`keys`, `haskey`, `length`, `values`, `isempty` and `is_transformed(vi)` on a `VarInfo` without a `VectorValueAccumulator` now throw an `ArgumentError` instead of reporting no variables. Record values with `VarInfo(VectorValueAccumulator(), DynamicPPL.default_accumulators()...)` in place of `VarInfo()`.

`VarNamedTuple` lookups and model evaluation throw `ArgumentError` for indices with a different number of dimensions, indices using `begin`, `end` or `:`, or Boolean masks of a different shape, in values set by index without a template; previously these could return the wrong element or be treated as missing. Supply a template or the whole array, or pass the `VarNamedTuple` itself rather than a `Dict` of its pairs ([#1544](https://github.com/TuringLang/DynamicPPL.jl/pull/1544)).

`marginalize` and the `DynamicPPLMarginalLogDensitiesExt` extension are removed in DynamicPPL 0.43. Users requiring the existing Turing/MLD integration can remain on DynamicPPL 0.42.x with a compatible Turing release—for example, Turing 0.49.0—and MarginalLogDensities 0.4.3–0.4.x. See the [0.42 marginalisation documentation](https://turinglang.org/DynamicPPL.jl/v0.42/api/#Marginalisation).

On newer DynamicPPL versions, MLD’s generic API requires a manually constructed log-density adapter; no drop-in model-level replacement is currently available. Migration of the integration into MLD is proposed in [MLD #47](https://github.com/ElOceanografo/MarginalLogDensities.jl/pull/47).
Expand Down
2 changes: 1 addition & 1 deletion docs/src/conditionfix.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ end

Note that in this case since the `VarNamedTuple` has no knowledge of the length or shape of `y`, DynamicPPL will assume that `y` is a `Base.Vector` of unknown length (hence the `GrowableArray` above).

This will work fine as long as `y` is indeed a `Base.Vector`.
This will work fine as long as `y` is indeed a `Base.Vector` and the model does not index it with `begin`, `end`, `:` or a Boolean mask.
However, if you want to avoid this, you should provide the full shape of `y` when defining the `VarNamedTuple`:

```@example 1
Expand Down
23 changes: 20 additions & 3 deletions src/chains.jl
Original file line number Diff line number Diff line change
Expand Up @@ -418,17 +418,34 @@ function predict(
return if include_all
prediction
else
# Re-key `params` with the model's templates, as `to_samples` does, so the model's
# leaves below match any index form in `params`; keep only the parts the model holds.
supplied = VarNamedTuple()
for (vn, value) in pairs(params)
# Must read `params` at the model's keys, not expand `value`: expanding reads
# unassigned elements and treats a `Tuple` as a single leaf.
leaves = if haskey(prediction.params, vn)
((vn, value),)
else
model_leaves = Iterators.filter(l -> subsumes(vn, l), keys(prediction.params))
((l, params[l]) for l in model_leaves if haskey(params, l))
end
for (leaf, leaf_value) in leaves
template = prediction.params.data[AbstractPPL.getsym(leaf)]
supplied = templated_setindex!!(supplied, leaf_value, leaf, template)
end
end
predicted_params = VarNamedTuple()
# Raw values retain sampled-variable boundaries before arrays are densified.
for (vn, value) in pairs(get_raw_values(vi))
leaves = AbstractPPL.varname_and_value_leaves(vn, value)
isempty(leaves) && haskey(params, vn) && !ismissing(params[vn]) && continue
isempty(leaves) && haskey(supplied, vn) && !ismissing(supplied[vn]) && continue
keep_all = all(
p -> !haskey(params, first(p)) || ismissing(params[first(p)]), leaves
p -> !haskey(supplied, first(p)) || ismissing(supplied[first(p)]), leaves
)
retained = keep_all ? ((vn, value),) : leaves
for (leaf, leaf_value) in retained
if keep_all || !haskey(params, leaf) || ismissing(params[leaf])
if keep_all || !haskey(supplied, leaf) || ismissing(supplied[leaf])
predicted_params = templated_setindex!!(
predicted_params,
leaf_value,
Expand Down
21 changes: 15 additions & 6 deletions src/varnamedtuple/getset.jl
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ function _getindex_optic(
return _getindex_optic(getindex(vnt.data, S), optic.child, orig_vn)
end
function _getindex_optic(pa::PartialArray, optic::AbstractPPL.Index, orig_vn)
_check_index_form(pa, optic.ix, orig_vn)
coptic = AbstractPPL.concretize_top_level(optic, pa.data)
child_value =
if _is_multiindex(pa, coptic.ix...; coptic.kw...) &&
Expand All @@ -77,12 +78,17 @@ end
return AbstractPPL.canview(head, x) && _haskey_optic(head(x), o.child)
end
function _haskey_optic(vnt::VarNamedTuple, name::VarName)
return _haskey_optic(vnt, AbstractPPL.varname_to_optic(name))
return _haskey_optic(vnt, AbstractPPL.varname_to_optic(name), name)
end
# `orig_vn` only names the variable in errors, as for `_getindex_optic`.
_haskey_optic(x, optic::AbstractPPL.AbstractOptic, orig_vn) = _haskey_optic(x, optic)
@inline _haskey_optic(@nospecialize(::Any), ::AbstractPPL.Iden) = true
@inline _haskey_optic(::VarNamedTuple, ::AbstractPPL.Index) = false
function _haskey_optic(vnt::VarNamedTuple, optic::AbstractPPL.Property{S}) where {S}
return Base.haskey(vnt.data, S) && _haskey_optic(getindex(vnt.data, S), optic.child)
function _haskey_optic(
vnt::VarNamedTuple, optic::AbstractPPL.Property{S}, orig_vn=@varname(_)
) where {S}
return Base.haskey(vnt.data, S) &&
_haskey_optic(getindex(vnt.data, S), optic.child, orig_vn)
end
function _haskey_optic(pa::PartialArray, ::AbstractPPL.Iden)
# no masked indices
Expand All @@ -96,17 +102,20 @@ function _haskey_optic(pa::PartialArray, ::AbstractPPL.Iden)
return true
end
end
function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index)
function _haskey_optic(pa::PartialArray, optic::AbstractPPL.Index, orig_vn=@varname(_))
_check_index_form(pa, optic.ix, orig_vn)
coptic = AbstractPPL.concretize_top_level(optic, pa.data)
# check the top level Index
Base.haskey(pa, coptic.ix...; coptic.kw...) || return false
# recurse if necessary
return if optic.child isa AbstractPPL.Iden
true
elseif _is_multiindex(pa, coptic.ix...; coptic.kw...)
_haskey_optic(_subset_partialarray(pa, coptic.ix...; coptic.kw...), optic.child)
_haskey_optic(
_subset_partialarray(pa, coptic.ix...; coptic.kw...), optic.child, orig_vn
)
else
_haskey_optic(getindex(pa, coptic.ix...; coptic.kw...), optic.child)
_haskey_optic(getindex(pa, coptic.ix...; coptic.kw...), optic.child, orig_vn)
end
end
function _haskey_optic(x::Tuple, optic::AbstractPPL.Index)
Expand Down
42 changes: 30 additions & 12 deletions src/varnamedtuple/partial_array.jl
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,36 @@ function _can_get_arraylikeblock(pa_data::AbstractArray)
return true
end

# Must throw, not report a miss, for an index that is resolved against the shape of storage
# built without a template (`begin`, `end`, `:`), a Boolean mask whose axes differ from the
# storage's, or an index with another number of dimensions: the element it names depends
# on the unknown shape.
function _check_index_form(pa::PartialArray, inds, vn)
pa.data isa GrowableArray || return nothing
n = 0
masks_fit = true
for ind in inds
if ind isa AbstractArray{Bool}
offset = n
masks_fit &= all(d -> axes(ind, d) == axes(pa.data, offset + d), 1:ndims(ind))
n += ndims(ind)
else
n += length(Base.index_ndims(ind))
end
end
resolved = !_has_colon_or_dynamicindex(inds)
resolved && masks_fit && n == ndims(pa) && return nothing
throw(
ArgumentError(
"Cannot look up `$vn` in values set with $(ndims(pa))-dimensional indices " *
"without a template: the element it names depends on the array's unknown " *
"shape. Supply a template or the whole array, or pass the `VarNamedTuple` " *
"itself rather than a `Dict` of its pairs" *
(resolved ? "; or use matching indices." : "."),
),
)
end

"""
Base.getindex(pa::PartialArray, inds::Vararg{Any}; kw...)

Expand Down Expand Up @@ -418,14 +448,6 @@ function Base.getindex(pa::PartialArray, inds::Vararg{Any}; kw...)
throw(alb_err)
end
elseif val isa GrowableArray
# This code path is hit for things like `vnt[@varname(x[:])]` where `x` is a PA that
# stores a GrowableArray. We warn the user that the result may be wrong.
# TODO(penelopeysm): The DynamicIndex check actually doesn't work here because if
# you index with `x[end]` for example, `inds` is already concretised outside of this
# function.
if any(ind -> ind isa AbstractPPL.DynamicIndex || ind isa Colon, inds)
_warn_growable_array_extraction()
end
return unwrap_internal_array(val)
else
return val
Expand Down Expand Up @@ -674,10 +696,6 @@ function BangBang.setindex!!(pa::PartialArray, value, inds::Vararg{Any}; kw...)
end

function _subset_partialarray(pa::PartialArray, inds::Vararg{Any}; kw...)
if pa.data isa GrowableArray &&
any(ind -> ind isa AbstractPPL.DynamicIndex || ind isa Colon, inds)
_warn_growable_array_extraction()
end
new_data = view(pa.data, inds...; kw...)
new_mask = view(pa.mask, inds...; kw...)
return PartialArray(new_data, new_mask)
Expand Down
3 changes: 3 additions & 0 deletions src/varnamedtuple/vnt.jl
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ efficient and type stable manner. It is mainly used through `getindex`, `setinde
shorthand for an identity `VarName`: `vnt[:x]` is equivalent to `vnt[@varname(x)]`.
Only symbols that are valid Julia identifiers are accepted. Use a `VarName` for indexed or
nested variables. `Symbol("x[1]")` and `Symbol("x.y")` are rejected rather than parsed.
`haskey` and `getindex` throw an `ArgumentError` for an index whose number of dimensions
differs from that of values set by index without a template, or that uses `begin`, `end`,
`:` or a Boolean mask of a different shape on such values.
Other notable methods are `merge` and `subset`.

`VarNamedTuple` has an ordering to its elements, and two `VarNamedTuple`s with the same keys
Expand Down
58 changes: 58 additions & 0 deletions test/model.jl
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,64 @@ const GDEMO_DEFAULT = DynamicPPL.TestUtils.demo_assume_observe_literal()
end
end

@model function linear_local()
y = zeros(2, 2)
y[1] ~ Normal()
return y[1]
end
@model function aliased_local()
y = zeros(3, 3)
y[4] ~ Normal()
return y[4]
end
@model function last_local()
y = zeros(3)
y[end] ~ Normal()
return y[3]
end
@model function masked_local()
y = zeros(3)
return y[[true, true, false]] ~ MvNormal(zeros(2), I)
end
@testset "values keyed in another index form without a template" begin
vi = VarInfo(RawValueAccumulator(false))
cartesian = Dict(@varname(y[1, 1]) => 1.0)
@test_throws ArgumentError init!!(
Xoshiro(1), linear_local(), vi, InitFromParams(cartesian, InitFromPrior())
)
untemplated = VarNamedTuple(pairs(cartesian))
@test_throws ArgumentError haskey(untemplated, @varname(y[1]))
@test_throws "`y[1]`" untemplated[@varname(y[1])]
@test haskey(untemplated, @varname(y[1, 1])) && !haskey(untemplated, @varname(y[2, 2]))
# Storage built from these keys is 2x2, where `y[4]` is `y[2, 2]` rather than `y[1, 2]`.
aliased = Dict(@varname(y[1, 2]) => 1.0, @varname(y[2, 2]) => 2.0)
@test_throws ArgumentError init!!(
Xoshiro(1), aliased_local(), vi, InitFromParams(aliased, nothing)
)
linear = VarNamedTuple(pairs(Dict(@varname(y[1]) => 1.0)))
@test isempty(DynamicPPL.predict(Xoshiro(1), linear_local(), linear).params)
whole = VarNamedTuple(; y=[1.0 0.0; 0.0 0.0])
@test isempty(DynamicPPL.predict(Xoshiro(1), linear_local(), whole).params)
# Storage built from these keys has length 2, where `end` names `y[2]`.
short = Dict(@varname(y[1]) => 1.0, @varname(y[2]) => 2.0)
@test_throws ArgumentError returned(last_local(), short)
@test_throws ArgumentError rand(Xoshiro(1), condition(last_local(), short))
@test_throws ArgumentError logjoint(condition(last_local(), short), VarNamedTuple())
@test_throws ArgumentError rand(Xoshiro(1), condition(masked_local(), short))
end

@model function first_of_two()
x = Vector{Vector{Float64}}(undef, 2)
return x[1] ~ MvNormal(zeros(2), I)
end
@testset "predict reads supplied values only where the model has variables" begin
xs = Vector{Vector{Float64}}(undef, 2)
xs[1] = [1.0, 2.0]
predicted(x) = DynamicPPL.predict(Xoshiro(1), first_of_two(), VarNamedTuple(; x)).params
@test isempty(predicted(xs))
@test isempty(predicted((xs[1],)))
end

@info "Completed $(@__FILE__) in $(now() - __now__)."

end # module
18 changes: 10 additions & 8 deletions test/varnamedtuple.jl
Original file line number Diff line number Diff line change
Expand Up @@ -965,14 +965,17 @@ end
x[2] := 2.0
end
warnmsg = r"Returning a `Base.Array` with"
# These should warn. There are various calls to _warn_growable_array() inside
# the VNT code to handle each of these cases.
@test_logs (:warn, warnmsg) vnt[@varname(x[:][1])]
@test_logs (:warn, warnmsg) vnt[@varname(x[:])]
@test_logs (:warn, warnmsg) vnt[@varname(x)]
@test_logs (:warn, warnmsg) vnt[@varname(x[begin])]
@test_logs (:warn, warnmsg) vnt[@varname(x[end])]
@test_logs (:warn, warnmsg) vnt[@varname(x[1:end])]
# Indices resolved against the presumed size throw instead.
for vn in (
@varname(x[:][1]),
@varname(x[:]),
@varname(x[begin]),
@varname(x[end]),
@varname(x[1:end]),
)
@test_throws ArgumentError vnt[vn]
end
# These shouldn't warn because they are safe
@test_logs vnt[@varname(x[1:2])]
@test_logs vnt[@varname(x[1])]
Expand Down Expand Up @@ -1096,7 +1099,6 @@ end
@test @inferred(getindex(vnt, @varname(x[1]))) == 1.0
# Check that we can get both indices
@test @inferred(getindex(vnt, @varname(x[1:2]))) == [1.0, 2.0]
@test @inferred(getindex(vnt, @varname(x[:]))) == [1.0, 2.0]
@test @inferred(getindex(vnt, @varname(x))) == [1.0, 2.0]

@test keys(vnt) == [@varname(x[1]), @varname(x[2])]
Expand Down
Loading