Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions src/api/gradients.jl
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ call.
"""
function gradient(f, input, cfg::GradientConfig = GradientConfig(input))
tape = GradientTape(f, input, cfg)
result = construct_result(input_hook(tape))
result = seeded_reverse_pass!(result, tape)
idle!(cfg.tape)
return result
try
result = construct_result(input_hook(tape))
return seeded_reverse_pass!(result, tape)
finally
idle!(cfg.tape)
end
end

"""
Expand All @@ -40,9 +42,11 @@ the returned value: `result = ReverseDiff.gradient!(result, f, input, cfg)`.
"""
function gradient!(result, f, input, cfg::GradientConfig = GradientConfig(input))
tape = GradientTape(f, input, cfg)
result = seeded_reverse_pass!(result, tape)
idle!(cfg.tape)
return result
try
return seeded_reverse_pass!(result, tape)
finally
idle!(cfg.tape)
end
end

###########################
Expand Down
46 changes: 28 additions & 18 deletions src/api/jacobians.jl
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,12 @@ call.
"""
function jacobian(f, input, cfg::JacobianConfig = JacobianConfig(input))
tape = JacobianTape(f, input, cfg)
isa(input, TrackedArray) && empty!(input.tape)
result = jacobian!(tape, input)
idle!(tape.tape)
return result
try
isa(input, TrackedArray) && empty!(input.tape)
return jacobian!(tape, input)
finally
idle!(cfg.tape)
end
end

"""
Expand All @@ -41,10 +43,12 @@ the returned value: `result = ReverseDiff.jacobian!(result, f, input, cfg)`.
"""
function jacobian!(result, f, input, cfg::JacobianConfig = JacobianConfig(input))
tape = JacobianTape(f, input, cfg)
isa(input, TrackedArray) && empty!(input.tape)
result = jacobian!(result, tape, input)
idle!(tape.tape)
return result
try
isa(input, TrackedArray) && empty!(input.tape)
return jacobian!(result, tape, input)
finally
idle!(cfg.tape)
end
end

#########################################################
Expand All @@ -59,11 +63,14 @@ form `f!(output::AbstractArray{<:Real}, input::AbstractArray{<:Real}...)`.
"""
function jacobian(f!, output, input, cfg::JacobianConfig = JacobianConfig(output, input))
tape = JacobianTape(f!, output, input, cfg)
isa(input, TrackedArray) && empty!(input.tape)
result = jacobian!(tape, input)
extract_result_value!(output, output_hook(tape))
idle!(tape.tape)
return result
try
isa(input, TrackedArray) && empty!(input.tape)
result = jacobian!(tape, input)
extract_result_value!(output, output_hook(tape))
return result
finally
idle!(cfg.tape)
end
end

"""
Expand All @@ -74,11 +81,14 @@ form `f!(output::AbstractArray{<:Real}, input::AbstractArray{<:Real}...)`.
"""
function jacobian!(result, f!, output, input, cfg::JacobianConfig = JacobianConfig(output, input))
tape = JacobianTape(f!, output, input, cfg)
isa(input, TrackedArray) && empty!(input.tape)
result = jacobian!(result, tape, input)
extract_result_value!(output, output_hook(tape))
idle!(tape.tape)
return result
try
isa(input, TrackedArray) && empty!(input.tape)
result = jacobian!(result, tape, input)
extract_result_value!(output, output_hook(tape))
return result
finally
idle!(cfg.tape)
end
end

###########################
Expand Down
81 changes: 43 additions & 38 deletions src/api/tape.jl
Original file line number Diff line number Diff line change
Expand Up @@ -211,20 +211,20 @@ element type and shape as `input`.
See `ReverseDiff.gradient` for a description of acceptable types for `input`.
"""
function GradientTape(f, input, cfg::GradientConfig = GradientConfig(input))
reopen!(cfg.tape)
track!(cfg.input, input)
tracked_ouput = f(cfg.input)
finish!(cfg.tape)
return _GradientTape(f, cfg.input, tracked_ouput, cfg.tape)
tracked_output = record_finished!(cfg.tape) do
track!(cfg.input, input)
f(cfg.input)
end
return _GradientTape(f, cfg.input, tracked_output, cfg.tape)
end

function GradientTape(f, input::Tuple, cfg::GradientConfig = GradientConfig(input))
reopen!(cfg.tape)
for i in eachindex(cfg.input)
track!(cfg.input[i], input[i])
tracked_output = record_finished!(cfg.tape) do
for i in eachindex(cfg.input)
track!(cfg.input[i], input[i])
end
f(cfg.input...)
end
tracked_output = f(cfg.input...)
finish!(cfg.tape)
return _GradientTape(f, cfg.input, tracked_output, cfg.tape)
end

Expand All @@ -245,20 +245,20 @@ element type and shape as `input`.
See `ReverseDiff.jacobian` for a description of acceptable types for `input`.
"""
function JacobianTape(f, input, cfg::JacobianConfig = JacobianConfig(input))
reopen!(cfg.tape)
track!(cfg.input, input)
tracked_ouput = f(cfg.input)
finish!(cfg.tape)
return _JacobianTape(f, cfg.input, tracked_ouput, cfg.tape)
tracked_output = record_finished!(cfg.tape) do
track!(cfg.input, input)
f(cfg.input)
end
return _JacobianTape(f, cfg.input, tracked_output, cfg.tape)
end

function JacobianTape(f, input::Tuple, cfg::JacobianConfig = JacobianConfig(input))
reopen!(cfg.tape)
for i in eachindex(cfg.input)
track!(cfg.input[i], input[i])
tracked_output = record_finished!(cfg.tape) do
for i in eachindex(cfg.input)
track!(cfg.input[i], input[i])
end
f(cfg.input...)
end
tracked_output = f(cfg.input...)
finish!(cfg.tape)
return _JacobianTape(f, cfg.input, tracked_output, cfg.tape)
end

Expand All @@ -275,22 +275,22 @@ element type and shape as `input`.
See `ReverseDiff.jacobian` for a description of acceptable types for `input`.
"""
function JacobianTape(f!, output, input, cfg::JacobianConfig = JacobianConfig(output, input))
reopen!(cfg.tape)
track!(cfg.output, output, cfg.tape)
track!(cfg.input, input)
f!(cfg.output, cfg.input)
finish!(cfg.tape)
record_finished!(cfg.tape) do
track!(cfg.output, output, cfg.tape)
track!(cfg.input, input)
f!(cfg.output, cfg.input)
end
return _JacobianTape(f!, cfg.input, cfg.output, cfg.tape)
end

function JacobianTape(f!, output, input::Tuple, cfg::JacobianConfig = JacobianConfig(output, input))
reopen!(cfg.tape)
track!(cfg.output, output, cfg.tape)
for i in eachindex(input)
track!(cfg.input[i], input[i])
record_finished!(cfg.tape) do
track!(cfg.output, output, cfg.tape)
for i in eachindex(input)
track!(cfg.input[i], input[i])
end
f!(cfg.output, cfg.input...)
end
f!(cfg.output, cfg.input...)
finish!(cfg.tape)
return _JacobianTape(f!, cfg.input, cfg.output, cfg.tape)
end

Expand All @@ -313,11 +313,16 @@ See `ReverseDiff.hessian` for a description of acceptable types for `input`.
function HessianTape(f, input, cfg::HessianConfig = HessianConfig(input))
gcfg = cfg.gradient_config
jcfg = cfg.jacobian_config
reopen!(jcfg.tape)
ht = _HessianTape(f, jcfg.input, similar(deriv(gcfg.input)), jcfg.tape)
track!(ht.input, input)
gt = GradientTape(f, ht.input, gcfg)
seeded_reverse_pass!(ht.output, gt.output, gt.input, gt.tape)
finish!(jcfg.tape)
return ht
return record_finished!(jcfg.tape) do
ht = _HessianTape(f, jcfg.input, similar(deriv(gcfg.input)), jcfg.tape)
track!(ht.input, input)
gt = GradientTape(f, ht.input, gcfg)
# the gradient tape is only needed for recording the Hessian
try
seeded_reverse_pass!(ht.output, gt.output, gt.input, gt.tape)
finally
idle!(gcfg.tape)
end
ht
end
end
19 changes: 14 additions & 5 deletions src/tape.jl
Original file line number Diff line number Diff line change
Expand Up @@ -79,16 +79,25 @@ function finish_if_new!(tp::InstructionTape)
return tp
end

# reopen an idle (finished and empty) tape for recording
function reopen!(tp::InstructionTape)
idle!(tp::InstructionTape) = finish!(empty!(tp))

# record `f()` onto `tp` and finish it; an idle tape is reopened, and is idle again if `f` throws
function record_finished!(f, tp::InstructionTape)
if !isrecording(tp) && isempty(tp.instructions)
empty!(tp)
out = try
f()
catch
idle!(tp)
rethrow()
end
else
out = f()
end
return tp
finish!(tp)
return out
end

idle!(tp::InstructionTape) = finish!(empty!(tp))

# undo the swap in `record!` so that the tape stays finished
@noinline function throw_finished(tp::InstructionTape)
@atomic tp.last = Finished()
Expand Down
59 changes: 59 additions & 0 deletions test/api/ConfigTests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,62 @@ end
@test ReverseDiff.hessian(sum_abs2, x, deepcopy(cfg)) == [2.0 0.0; 0.0 2.0]
@test ReverseDiff.hessian(sum_abs2, x, cfg) == [2.0 0.0; 0.0 2.0]
end

##############
# exceptions #
##############

# throws after some instructions were recorded
function throw_if_large(x)
y = sum(abs2, x)
if y > 10
throw(DomainError(y))
end
return x
end
sum_abs2_if_small(x) = sum_abs2(throw_if_large(x))
double_if_small(x) = double(throw_if_large(x))

# throws in the reverse pass
sum_failing_pullback(x) = sum(x)
sum_failing_pullback(x::ReverseDiff.TrackedVector) = ReverseDiff.track(sum_failing_pullback, x)
ReverseDiff.@grad function sum_failing_pullback(x::AbstractVector)
return sum(ReverseDiff.value(x)), Δ -> throw(DomainError(Δ))
end
sums_failing_pullback(x) = [sum_failing_pullback(x)]

@testset "a config can be used and copied after recording throws" begin
small, large = [1.0, 2.0], [3.0, 4.0]

cfg = GradientConfig(small)
@test_throws DomainError ReverseDiff.gradient(sum_abs2_if_small, large, cfg)
@test ReverseDiff.gradient(sum_abs2_if_small, small, deepcopy(cfg)) == [2.0, 4.0]
@test ReverseDiff.gradient(sum_abs2_if_small, small, cfg) == [2.0, 4.0]

cfg = JacobianConfig(small)
@test_throws DomainError ReverseDiff.jacobian(double_if_small, large, cfg)
@test ReverseDiff.jacobian(double_if_small, small, deepcopy(cfg)) == [2.0 0.0; 0.0 2.0]
@test ReverseDiff.jacobian(double_if_small, small, cfg) == [2.0 0.0; 0.0 2.0]

cfg = HessianConfig(small)
@test_throws DomainError ReverseDiff.hessian(sum_abs2_if_small, large, cfg)
@test_throws DomainError ReverseDiff.HessianTape(sum_abs2_if_small, large, cfg)
@test ReverseDiff.hessian(sum_abs2_if_small, small, deepcopy(cfg)) == [2.0 0.0; 0.0 2.0]
@test ReverseDiff.hessian(sum_abs2_if_small, small, cfg) == [2.0 0.0; 0.0 2.0]
end

@testset "a config can be used after the reverse pass throws" begin
x = [1.0, 2.0]

cfg = GradientConfig(x)
@test_throws DomainError ReverseDiff.gradient(sum_failing_pullback, x, cfg)
@test ReverseDiff.gradient(sum_abs2, x, cfg) == [2.0, 4.0]

cfg = JacobianConfig(x)
@test_throws DomainError ReverseDiff.jacobian(sums_failing_pullback, x, cfg)
@test ReverseDiff.jacobian(double, x, cfg) == [2.0 0.0; 0.0 2.0]

cfg = HessianConfig(x)
@test_throws DomainError ReverseDiff.HessianTape(sum_failing_pullback, x, cfg)
@test ReverseDiff.hessian(sum_abs2, x, cfg) == [2.0 0.0; 0.0 2.0]
end
20 changes: 20 additions & 0 deletions test/api/JacobianTests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,26 @@ f251(x) = x .^ 2
end
end

prod2!(y, a, b) = (y .= a .* b; y)

@testset "`f!(y, a, b)`" begin
a, b = [1.0, 2.0], [3.0, 4.0]
Ja, Jb = [3.0 0.0; 0.0 4.0], [1.0 0.0; 0.0 2.0]

y = zeros(2)
@test ReverseDiff.jacobian(prod2!, y, (a, b)) == (Ja, Jb)
@test y == [3.0, 8.0]

results = (zeros(2, 2), zeros(2, 2))
ReverseDiff.jacobian!(results, prod2!, zeros(2), (a, b))
@test results == (Ja, Jb)

tape = ReverseDiff.JacobianTape(prod2!, zeros(2), (zeros(2), zeros(2)))
for t in (tape, ReverseDiff.compile(tape))
@test ReverseDiff.jacobian!(t, (a, b)) == (Ja, Jb)
end
end

@testset "outputs whose element type is not `Real`" begin
@test ReverseDiff.jacobian(x -> Any[2 * x[1], 3 * x[2]], [1.0, 2.0]) == [2.0 0.0; 0.0 3.0]
@test ReverseDiff.jacobian(x -> [1.0, 2.0], rand(3)) == zeros(2, 3)
Expand Down
Loading