diff --git a/Project.toml b/Project.toml index 396b3f7f..36bccf17 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ForwardDiff" uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "1.4.7" +version = "1.5.1" [deps] CommonSubexpressions = "bbf7d656-a473-5ed7-a52c-81e309532950" diff --git a/src/dual.jl b/src/dual.jl index 48d5d0e3..d54d4c07 100644 --- a/src/dual.jl +++ b/src/dual.jl @@ -305,8 +305,10 @@ end Base.copy(d::Dual) = d -Base.eps(d::Dual) = eps(value(d)) -Base.eps(::Type{D}) where {D<:Dual} = eps(valtype(D)) +for op in (:eps, :typemin, :typemax, :floatmin, :floatmax, :maxintfloat) + @eval Base.$op(d::Dual) = $op(d.value) + @eval Base.$op(::Type{<:Dual{T,V}}) where {T,V} = $op(V) +end # The `base` keyword was added in Julia 1.8: # https://github.com/JuliaLang/julia/pull/42428 @@ -875,12 +877,52 @@ function SpecialFunctions.logabsgamma(d::Dual{T,<:Real}) where {T} end # Derivatives wrt to first parameter and precision setting are not supported -function SpecialFunctions.gamma_inc(a::Real, d::Dual{T,<:Real}, ind::Integer) where {T} - x = value(d) +function SpecialFunctions.gamma_inc(a::Real, d::Dual{T}, ind::Integer) where {T} + x = d.value p, q = SpecialFunctions.gamma_inc(a, x, ind) - ∂p = exp(-x) * x^(a - 1) / SpecialFunctions.gamma(a) * partials(d) + ∂p = _gamma_inc_dx(a, x) * d.partials return (Dual{T}(p, ∂p), Dual{T}(q, -∂p)) end +SpecialFunctions.gamma_inc(a::Dual, x::Real, ind::Integer) = _gamma_inc_constant_a(a, x, ind) +SpecialFunctions.gamma_inc(a::Dual, x::Dual, ind::Integer) = _gamma_inc_constant_a(a, x, ind) +function _gamma_inc_constant_a(a::Dual, x::Real, ind::Integer) + iszero(a.partials) || throw(ArgumentError("derivatives with respect to the first argument `a` are not supported")) + return SpecialFunctions.gamma_inc(a.value, x, ind) +end + +# TODO: Remove fallback once SpecialFunctions >= 2.10 is required +if isdefined(SpecialFunctions, :gamma_inc_dx) + _gamma_inc_dx(a, x) = SpecialFunctions.gamma_inc_dx(a, x) + # The first argument `x` is the shape `a`, the second argument `y` is `x` + @define_binary_dual_op( + SpecialFunctions.gamma_inc_dx, + begin + va, vx = value(Txy, x), value(Txy, y) + f = SpecialFunctions.gamma_inc_dx(va, vx) + ∂f∂a = SpecialFunctions._gamma_inc_dx_da(va, vx, f) + ∂f∂x = SpecialFunctions._gamma_inc_dx_dx(va, vx, f) + return dual_definition_retval(Val{Txy}(), f, ∂f∂a, partials(Txy, x), ∂f∂x, partials(Txy, y)) + end, + begin + va = value(Tx, x) + f = SpecialFunctions.gamma_inc_dx(va, y) + ∂f∂a = SpecialFunctions._gamma_inc_dx_da(va, y, f) + return dual_definition_retval(Val{Tx}(), f, ∂f∂a, partials(Tx, x)) + end, + begin + vx = value(Ty, y) + f = SpecialFunctions.gamma_inc_dx(x, vx) + ∂f∂x = SpecialFunctions._gamma_inc_dx_dx(x, vx, f) + return dual_definition_retval(Val{Ty}(), f, ∂f∂x, partials(Ty, y)) + end, + ) +else + function _gamma_inc_dx(a, x) + f = exp(-x) * x^(a - 1) / SpecialFunctions.gamma(a) + # avoid overflow for large `a` + return isfinite(f) || iszero(x) ? f : exp((a - 1) * log(x) - x - SpecialFunctions.loggamma(a)) + end +end ################### # Pretty Printing # @@ -894,10 +936,4 @@ function Base.show(io::IO, d::Dual{T,V,N}) where {T,V,N} print(io, ")") end -for op in (:(Base.typemin), :(Base.typemax), :(Base.floatmin), :(Base.floatmax)) - @eval function $op(::Type{ForwardDiff.Dual{T,V,N}}) where {T,V,N} - ForwardDiff.Dual{T,V,N}($op(V)) - end -end - Printf.tofloat(d::Dual) = Printf.tofloat(value(d)) diff --git a/src/prelude.jl b/src/prelude.jl index 9e037afa..3d2eef91 100644 --- a/src/prelude.jl +++ b/src/prelude.jl @@ -3,7 +3,7 @@ const DEFAULT_CHUNK_THRESHOLD = @load_preference("default_chunk_threshold", 12) const AMBIGUOUS_TYPES = (AbstractFloat, Irrational, Integer, Rational, Real, RoundingMode) -const UNARY_PREDICATES = Symbol[:isinf, :isnan, :isfinite, :iseven, :isodd, :isreal, :isinteger] +const UNARY_PREDICATES = Symbol[:isinf, :isnan, :isfinite, :issubnormal, :iseven, :isodd, :isreal, :isinteger] struct Chunk{N} end diff --git a/test/DualTest.jl b/test/DualTest.jl index e7429be5..e9911a46 100644 --- a/test/DualTest.jl +++ b/test/DualTest.jl @@ -10,6 +10,7 @@ using NaNMath, SpecialFunctions, LogExpFunctions using DiffRules import Calculus +import DiffResults import LinearAlgebra struct TestTag end @@ -383,6 +384,12 @@ ForwardDiff.:≺(::Type{OuterTestTag}, ::Type{TestTag}) = false @test isinf(Dual{TestTag}(Dual{TestTag}(Inf, M_PARTIALS), NESTED_PARTIALS)) @test !(isinf(NESTED_FDNUM)) + @test issubnormal(Dual{TestTag}(1.0f-38, PARTIALS)) + @test !(issubnormal(Dual{TestTag}(1.0f-37, PARTIALS))) + + @test issubnormal(Dual{TestTag}(Dual{TestTag}(1.0f-38, M_PARTIALS), NESTED_PARTIALS)) + @test !(issubnormal(Dual{TestTag}(Dual{TestTag}(1.0f-37, M_PARTIALS), NESTED_PARTIALS))) + @test isreal(FDNUM) @test isreal(NESTED_FDNUM) @@ -681,25 +688,15 @@ end @test iszero(ForwardDiff.partials(ForwardDiff.partials(x^y)[1])) end -@testset "Type min/max" begin - d1 = Dual(1.0) - dinf = typemax(typeof(d1)) - dminf = typemin(typeof(d1)) - @test dminf < d1 < dinf - @test typeof(dminf) === typeof(d1) - @test typeof(dinf) === typeof(d1) - @test !isfinite(dminf) - @test !isfinite(dinf) - - dfmin = floatmin(typeof(d1)) - dfmax = floatmax(typeof(d1)) - @test dfmin < d1 < dfmax - @test typeof(dfmin) === typeof(d1) - @test typeof(dfmax) === typeof(d1) - @test isfinite(dfmin) - @test isfinite(dfmax) - - @test floatmin(Dual{Nothing, ForwardDiff.Dual{Nothing, Float64, 2}, 1}) === Dual{Nothing}(Dual{Nothing}(floatmin(Float64),0.0,0.0),Dual{Nothing}(0.0,0.0,0.0)) +@testset "Type min/max: $T" for T in (Float32, Float64) + d = Dual(T(1), T(2)) + nested = Dual(Dual(T(1), T(2), T(3)), Dual(T(4), T(5), T(6))) + for f in (typemin, typemax, floatmin, floatmax, maxintfloat) + @test f(d) === f(T) + @test f(typeof(d)) === f(T) + @test f(nested) === f(T) + @test f(typeof(nested)) === f(T) + end end @testset "Integer" begin @@ -734,6 +731,107 @@ end @test ForwardDiff.derivative(x -> sum(1 .+ x .* (0:0.1:1)), 1) == 5.5 end +@testset "gamma_inc" begin + # Accurate derivatives for large `a` require SpecialFunctions >= 2.10 + has_gamma_inc_dx = isdefined(SpecialFunctions, :gamma_inc_dx) + # ∂/∂x gamma_inc(a, x)[1] = x^(a - 1) e^(-x) / Γ(a) + ∂p(a, x) = exp((big(a) - 1) * log(big(x)) - big(x)) / gamma(big(a)) + + for V in (Float32, Float64), (a, x) in ((2.5, 1.0), (150, 150), (200, 180), (1e6, 1e6)), ind in 0:2 + dp = ForwardDiff.derivative(x -> gamma_inc(V(a), x, ind)[1], V(x)) + dq = ForwardDiff.derivative(x -> gamma_inc(V(a), x, ind)[2], V(x)) + @test dp isa V + @test dp ≈ ∂p(a, x) rtol = 4 * eps(V) broken = !has_gamma_inc_dx && a > 100 + @test isequal(dq, -dp) + end + + d2p(a, x) = ForwardDiff.derivative(x -> ForwardDiff.derivative(x -> gamma_inc(a, x, 0)[1], x), x) + d3p(a, x) = ForwardDiff.derivative(x -> d2p(a, x), x) + for (a, x) in ((150.0, 150.0), (1e6, 1e6), (1e6, 999_000.0), (3.0, 1e-160)) + @test @inferred(d2p(a, x)) ≈ ∂p(a, x) * (big(a) - 1 - big(x)) / big(x) rtol = 8 * eps() broken = !has_gamma_inc_dx && a > 100 + end + @test ForwardDiff.derivative(x -> gamma_inc(1.0, x, 0)[1], 0.0) == 1 + @test d2p(2.0, 0.0) == 1 + @test d3p(3.0, 0.0) == 1 + + # Infinite derivative at zero + for f in ( + x -> gamma_inc(0.5, zero(x), 0)[1], + x -> gamma_inc(0.5, zero(x), 0)[2], + y -> ForwardDiff.derivative(x -> gamma_inc(0.5, zero(x), 0)[1], y), + ) + df = ForwardDiff.derivative(f, 1.0) + @test ForwardDiff.NANSAFE_MODE_ENABLED ? iszero(df) : isnan(df) + end + + # Duals with zero partials are constants + a = Dual{TestTag}(2.0, 0.0) + x = Dual{TestTag}(1.5, 1.0) + @test gamma_inc(a, 1.5, 0) === gamma_inc(2.0, 1.5, 0) + @test gamma_inc(a, x, 0) === gamma_inc(2.0, x, 0) + @test gamma_inc(Dual{OuterTestTag}(a, zero(a)), x, 0) === gamma_inc(2.0, x, 0) + msg = "ArgumentError: derivatives with respect to the first argument `a` are not supported" + @test_throws msg ForwardDiff.derivative(a -> gamma_inc(a, 1.5, 0)[1], 2.0) + @test_throws msg ForwardDiff.derivative(a -> ForwardDiff.derivative(x -> gamma_inc(a, x, 0)[1], 1.5), 2.0) +end + +@testset "gamma_inc_dx" begin + if isdefined(SpecialFunctions, :gamma_inc_dx) + # The x-derivative of `gamma_inc` is `gamma_inc_dx` + for (a, x) in ((2.5, 1.0), (150.0, 150.0)) + @test ForwardDiff.derivative(x -> gamma_inc(a, x, 0)[1], x) === SpecialFunctions.gamma_inc_dx(a, x) + end + + # f = x^(a - 1) e^(-x) / Γ(a), ∂f/∂a = f * (log(x) - ψ(a)), ∂f/∂x = f * (a - 1 - x) / x + function ∇f(a, x) + a, x = big(a), big(x) + l = (a - 1) * log(x) - x + e = exp(l) + # At the poles of ψ(a), f = 0 and ∂f/∂a = (-1)^a Γ(1 - a) x^(a - 1) e^(-x) + isinteger(a) && a <= 0 && return [cospi(a) * e * gamma(1 - a), zero(a)] + # 1 / Γ(a) = sin(πa) Γ(1 - a) / π + f = a > 0 ? exp(l - loggamma(a)) : e * sinpi(a) * gamma(1 - a) / big(π) + return [f * (log(x) - digamma(a)), f * (a - 1 - x) / x] + end + function ∇²f(a, x) + setprecision(BigFloat, 512) do + h = big(1e-40) + return hcat((∇f(a + h, x) - ∇f(a - h, x)) / 2h, (∇f(a, x + h) - ∇f(a, x - h)) / 2h) + end + end + + for V in (Float32, Float64), (a, x) in ((2.5, 1.0), (0.3, 2.0), (-2.0, 1.5), (150.0, 150.0), (150.0, 149.5), (1e6, 1e6), (1e10, 1e10)) + ∂a, ∂x = setprecision(() -> V.(∇f(V(a), V(x))), BigFloat, 512) + dfda = ForwardDiff.derivative(a -> SpecialFunctions.gamma_inc_dx(a, V(x)), V(a)) + dfdx = ForwardDiff.derivative(x -> SpecialFunctions.gamma_inc_dx(V(a), x), V(x)) + @test dfda isa V + @test dfda ≈ ∂a rtol = 8 * eps(V) + @test dfdx isa V + @test dfdx ≈ ∂x rtol = 8 * eps(V) + @test ForwardDiff.gradient(v -> SpecialFunctions.gamma_inc_dx(v[1], v[2]), [V(a), V(x)]) == [dfda, dfdx] + end + + # `a` and `x` with different tags + d2fdadx(a, x) = ForwardDiff.derivative(a -> ForwardDiff.derivative(x -> SpecialFunctions.gamma_inc_dx(a, x), x), a) + d2fdxda(a, x) = ForwardDiff.derivative(x -> ForwardDiff.derivative(a -> SpecialFunctions.gamma_inc_dx(a, x), a), x) + for (a, x) in ((2.5, 1.0), (0.3, 2.0), (-2.0, 1.5), (150.0, 150.0), (1e6, 1e6), (1e10, 1e10)) + H = Float64.(∇²f(a, x)) + result = ForwardDiff.hessian!(DiffResults.HessianResult([a, x]), v -> SpecialFunctions.gamma_inc_dx(v[1], v[2]), [a, x]) + # The gradient is computed with nested duals + @test DiffResults.gradient(result) ≈ setprecision(() -> Float64.(∇f(a, x)), BigFloat, 512) rtol = 8 * eps() + @test DiffResults.hessian(result) ≈ H rtol = 1e-14 + @test @inferred(d2fdadx(a, x)) ≈ H[1, 2] rtol = 1e-14 + @test d2fdxda(a, x) ≈ H[2, 1] rtol = 1e-14 + end + else + # The x-derivative of `gamma_inc` falls back to the closed form + for (a, x) in ((2.5, 1.0), (30.0, 25.0)) + @test ForwardDiff.derivative(x -> gamma_inc(a, x, 0)[1], x) === exp(-x) * x^(a - 1) / gamma(a) + end + @test ForwardDiff.derivative(x -> gamma_inc(150.0, x, 0)[1], 150.0) ≈ exp(149 * log(big(150)) - 150) / gamma(big(150)) rtol = 1e-12 + end +end + @testset "Givens rotations: consistency with `LinearAlgebra.givensAlgorithm` for zero partials (no duals)" begin # Test different branches in `LinearAlgebra.givensAlgorithm` for f in [randexp(), -randexp()], g in [0.0, f / 2, 2f, -f / 2, -2f]