Enable testing correctness against reference, reorganize testing (#103)

* Start reference tests

* Continue references

* Restructure test suite

* No timing

* Add Documenter

* Qualify doctest

* Fix correctness

* Fix docs

* Refix

* fIX NESTED

Project.toml

@@ -37,7 +37,6 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 DifferentiationInterfaceChainRulesCoreExt = "ChainRulesCore"
 DifferentiationInterfaceChairmarksExt = ["Chairmarks"]
 DifferentiationInterfaceComponentArraysExt = ["ComponentArrays"]
-DifferentiationInterfaceCorrectnessTestExt = ["ForwardDiff", "Zygote"]
 DifferentiationInterfaceDiffractorExt = ["AbstractDifferentiation", "Diffractor"]
 DifferentiationInterfaceEnzymeExt = "Enzyme"
 DifferentiationInterfaceFastDifferentiationExt = ["FastDifferentiation", "RuntimeGeneratedFunctions"]

docs/src/tutorial.md

@@ -129,25 +129,31 @@ It's blazingly fast.
 And you know what's even better?
 You didn't need to look at the docs of either ForwardDiff.jl or Enzyme.jl to achieve top performance with both, or to compare them.
 
-## Benchmarking
+## Testing and benchmarking
 
-DifferentiationInterface.jl also provides some benchmarking utilities, for more involved comparison between backends.
+DifferentiationInterface.jl also provides some utilities for more involved comparison between backends.
 They are gathered in a submodule.
 
 ```@repl tuto
 using DifferentiationInterface.DifferentiationTest
 ```
 
-We can now use [`test_differentiation`](@ref) in benchmarking mode, and convert its output to a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl) (disregard the test logging):
+The main entry point is [`test_differentiation`](@ref), which is used as follows:
 
 ```@repl tuto
 data = test_differentiation(
     [AutoForwardDiff(), AutoEnzyme(Enzyme.Reverse)],  # backends to compare
     [gradient],  # operators to try
     [Scenario(f; x=x)];  # test scenario
-    correctness=false,  # we don't care about checking the result
-    benchmark=true,  # we care about measuring
+    correctness=AutoZygote(),  # compare results to a "ground truth" from Zygote
+    benchmark=true,  # measure runtime and allocations too
+    detailed=true,  # print detailed test set
 );
+```
+
+The output of `test_differentiation` when `benchmark=true` can be converted to a `DataFrame` from [DataFrames.jl](https://github.com/JuliaData/DataFrames.jl):
+
+```@repl tuto
 df = DataFrames.DataFrame(pairs(data)...)
 ```
 

ext/DifferentiationInterfaceChairmarksExt/DifferentiationInterfaceChairmarksExt.jl

@@ -155,7 +155,7 @@ function run_benchmark!(
 )
     (; f, x, y) = deepcopy(scen)
     extras = prepare_jacobian(f, ba, x)
-    jac_template = similar(y, length(y), length(x))
+    jac_template = Matrix{eltype(y)}(undef, length(y), length(x))
     bench1 = @be mysimilar(jac_template) value_and_jacobian!!(f, _, ba, x, extras)
     # never test allocations
     record!(data, ba, op, value_and_jacobian!!, scen, bench1)
@@ -172,7 +172,7 @@ function run_benchmark!(
     (; f, x, y) = deepcopy(scen)
     f! = f
     extras = prepare_jacobian(f!, ba, y, x)
-    jac_template = similar(y, length(y), length(x))
+    jac_template = Matrix{eltype(y)}(undef, length(y), length(x))
     bench1 = @be (mysimilar(y), mysimilar(jac_template)) value_and_jacobian!!(
         f!, _[1], _[2], ba, x, extras
     )

ext/DifferentiationInterfaceComponentArraysExt/DifferentiationInterfaceComponentArraysExt.jl

@@ -1,26 +1,64 @@
 module DifferentiationInterfaceComponentArraysExt
 
 using ComponentArrays: ComponentVector
-using DifferentiationInterface.DifferentiationTest: Scenario, SCALING_VEC
+using DifferentiationInterface.DifferentiationTest:
+    Scenario, Reference, make_scalar_to_array, make_scalar_to_array!, scalar_to_array_ref
+using LinearAlgebra: dot
 
-const SCALING_CVEC = ComponentVector(; a=collect(1:7), b=collect(8:12))
-
-function scalar_to_componentvector(x::Number)::ComponentVector
-    return sin.(SCALING_CVEC .* x) # output size 12
-end
+## Vector to scalar
 
 function componentvector_to_scalar(x::ComponentVector)::Number
     return sum(sin, x.a) + sum(cos, x.b)
 end
 
+componentvector_to_scalar_gradient(x) = ComponentVector(; a=cos.(x.a), b=-sin.(x.b))
+
+function componentvector_to_scalar_pushforward(x, dx)
+    return dot(componentvector_to_scalar_gradient(x), dx)
+end
+
+function componentvector_to_scalar_pullback(x, dy)
+    return componentvector_to_scalar_gradient(x) .* dy
+end
+
+function componentvector_to_scalar_ref()
+    return Reference(;
+        pushforward=componentvector_to_scalar_pushforward,
+        pullback=componentvector_to_scalar_pullback,
+        gradient=componentvector_to_scalar_gradient,
+    )
+end
+
+## Gather
+
+const SCALING_CVEC = ComponentVector(; a=collect(1:7), b=collect(8:12))
+
 function component_scenarios_allocating()
     return [
-        Scenario(scalar_to_componentvector; x=2.0),
+        Scenario(
+            make_scalar_to_array(SCALING_CVEC); x=2.0, ref=scalar_to_array_ref(SCALING_CVEC)
+        ),
         Scenario(
             componentvector_to_scalar;
             x=ComponentVector{Float64}(; a=collect(1:7), b=collect(8:12)),
+            ref=componentvector_to_scalar_ref(),
         ),
     ]
 end
 
+function component_scenarios_mutating()
+    return [
+        Scenario(
+            make_scalar_to_array!(SCALING_CVEC);
+            x=2.0,
+            y=float.(SCALING_CVEC),
+            ref=scalar_to_array_ref(SCALING_CVEC),
+        ),
+    ]
+end
+
+function component_scenarios()
+    return vcat(component_scenarios_allocating(), component_scenarios_mutating())
+end
+
 end