From 3356c7db6750b1bb80d911d346b0a6e3b947c514 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 3 Feb 2021 09:33:54 +0000 Subject: [PATCH 01/26] Introduce InputTrait and export --- Project.toml | 2 +- src/Models.jl | 1 + src/traits.jl | 30 ++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5dd0509..28b1b43 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Models" uuid = "e6388cff-ecff-480c-9b53-83211bf7812a" authors = ["Invenia Technical Computing Corporation"] -version = "0.2.4" +version = "0.3.0" [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/src/Models.jl b/src/Models.jl index 816c8df..5997bf1 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -6,6 +6,7 @@ export Model, Template export fit, predict, submodels, estimate_type, output_type export EstimateTrait, PointEstimate, DistributionEstimate export OutputTrait, SingleOutput, MultiOutput +export InputTrait, PointInput, DistributionInput """ Template diff --git a/src/traits.jl b/src/traits.jl index 915716a..29401e0 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -58,3 +58,33 @@ Return the [`OutputTrait`] of the [`Model`](@ref) or [`Template`](@ref). """ output_type(::T) where T = output_type(T) output_type(T::Type) = throw(MethodError(output_type, (T,))) # to prevent recursion + +""" + InputTrait + +The `InputTrait` specifies if the model supports point or distribution inputs to predict, +denoted by [`PointInput`](@ref) or [`DistributionInput`](@ref), respectively. +""" +abstract type InputTrait end + +""" + PointInput <: InputTrait + +Specifies that the [`Model`](@ref) accepts real-valued input variables to `predict`. +""" +abstract type PointInput <: InputTrait end + +""" + DistributionInput <: InputTrait + +Specifies that the [`Model`](@ref) returns a distribution over the input vairables to `predict`. +""" +abstract type DistributionInput <: InputTrait end + +""" + input_type(::T) where T = input_type(T) + +Return the [`InputTrait`] of the [`Model`](@ref) or [`Template`](@ref). +""" +input_type(::T) where T = input_type(T) +input_type(T::Type) = throw(MethodError(input_type, (T,))) # to prevent recursion From c4d43c9d04cb4e4f8c455d151c95a2185e1670a5 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 4 Feb 2021 14:02:32 +0000 Subject: [PATCH 02/26] Fix docstring --- src/traits.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/traits.jl b/src/traits.jl index 29401e0..b4e5d10 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -77,7 +77,7 @@ abstract type PointInput <: InputTrait end """ DistributionInput <: InputTrait -Specifies that the [`Model`](@ref) returns a distribution over the input vairables to `predict`. +Specifies that the [`Model`](@ref) accepts a distribution over the input variables to `predict`. """ abstract type DistributionInput <: InputTrait end From c309ddf151f183a79a527aa44811d134100e9ca4 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Mon, 15 Feb 2021 11:24:32 +0000 Subject: [PATCH 03/26] Rename InputTrait -> InjectTrait --- src/Models.jl | 4 ++-- src/traits.jl | 35 +++++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/Models.jl b/src/Models.jl index 5997bf1..89fa1a5 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -3,10 +3,10 @@ module Models import StatsBase: fit, predict export Model, Template -export fit, predict, submodels, estimate_type, output_type +export fit, predict, submodels, estimate_type, output_type, inject_type export EstimateTrait, PointEstimate, DistributionEstimate export OutputTrait, SingleOutput, MultiOutput -export InputTrait, PointInput, DistributionInput +export InjectTrait, PointInject, DistributionInject, PointOrDistributionInject """ Template diff --git a/src/traits.jl b/src/traits.jl index b4e5d10..6e15a01 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -60,31 +60,42 @@ output_type(::T) where T = output_type(T) output_type(T::Type) = throw(MethodError(output_type, (T,))) # to prevent recursion """ - InputTrait + InjectTrait -The `InputTrait` specifies if the model supports point or distribution inputs to predict, -denoted by [`PointInput`](@ref) or [`DistributionInput`](@ref), respectively. +The `InjectTrait` specifies if the model supports point or distribution injections to predict, +denoted by [`PointInject`](@ref) or [`DistributionInject`](@ref), respectively. """ -abstract type InputTrait end +abstract type InjectTrait end """ - PointInput <: InputTrait + PointInject <: InjectTrait Specifies that the [`Model`](@ref) accepts real-valued input variables to `predict`. """ -abstract type PointInput <: InputTrait end +abstract type PointInject <: InjectTrait end """ - DistributionInput <: InputTrait + DistributionInject <: InjectTrait Specifies that the [`Model`](@ref) accepts a distribution over the input variables to `predict`. """ -abstract type DistributionInput <: InputTrait end +abstract type DistributionInject <: InjectTrait end """ - input_type(::T) where T = input_type(T) + PointOrDistributionInject <: InjectTrait -Return the [`InputTrait`] of the [`Model`](@ref) or [`Template`](@ref). +Specifies that the [`Model`](@ref) accepts real-values or a distribution over the input +variables to `predict`. """ -input_type(::T) where T = input_type(T) -input_type(T::Type) = throw(MethodError(input_type, (T,))) # to prevent recursion +abstract type PointOrDistributionInject <: InjectTrait end + +""" + inject_type(::T) where T = inject_type(T) + +Return the [`InjectTrait`] of the [`Model`](@ref) or [`Template`](@ref). +""" +inject_type(::T) where T = inject_type(T) +inject_type(T::Type) = throw(MethodError(inject_type, (T,))) # to prevent recursion + +inject_type(::Type{<:Model}) = PointInject +inject_type(::Type{<:Template}) = PointInject From 18973e7d2fd856ac5d062c5ef6544dcc2a98477b Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 16 Feb 2021 16:20:32 +0000 Subject: [PATCH 04/26] Inject trait tests --- Project.toml | 2 +- test/traits.jl | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 28b1b43..95e5a2b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Models" uuid = "e6388cff-ecff-480c-9b53-83211bf7812a" authors = ["Invenia Technical Computing Corporation"] -version = "0.3.0" +version = "0.2.5" [deps] Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" diff --git a/test/traits.jl b/test/traits.jl index 12be8c0..c8662dd 100644 --- a/test/traits.jl +++ b/test/traits.jl @@ -5,7 +5,7 @@ struct DummyModel <: Model end estimates = (PointEstimate, DistributionEstimate) outputs = (SingleOutput, MultiOutput) - + @testset "$est, $out" for (est, out) in Iterators.product(estimates, outputs) @testset "Errors if traits are not defined" begin @@ -29,5 +29,20 @@ struct DummyModel <: Model end end + @testset "Inject Trait" begin + + @testset "Default" begin + @test inject_type(DummyTemplate) == inject_type(DummyModel) == PointInject + end + + injects = (DistributionInject, PointOrDistributionInject) + + @testset "$inj is defined" for inj in injects + inject_type(m::Type{<:DummyTemplate}) = inj + inject_type(m::Type{<:DummyModel}) = inj + + @test inject_type(DummyTemplate) == inject_type(DummyModel) == inj + end + end end From b70ad2c3f8bcc1859611fe3a0ec6a8ddcd63153d Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Feb 2021 09:39:31 +0000 Subject: [PATCH 05/26] Add method and tests for vector vector predict --- src/Models.jl | 5 +++++ src/test_utils.jl | 22 ++++++++++++++-------- test/test_utils.jl | 5 +++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Models.jl b/src/Models.jl index 89fa1a5..2baded0 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -39,6 +39,7 @@ function fit end """ predict(model::Model, inputs::AbstractMatrix) + predict(model::Model, inputs::AbstractVector{<:AbstractVector}) Predict targets for the provided the collection of `inputs` and [`Model`](@ref). @@ -50,6 +51,10 @@ return a `AbstractVector{<:Distribution}`. """ function predict end +function predict(model::Model, inputs::AbstractVector{<:AbstractVector}) + return predict(model, hcat(inputs...)) +end + """ submodels(::Union{Template, Model}) diff --git a/src/test_utils.jl b/src/test_utils.jl index e33d173..dab1345 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -36,7 +36,7 @@ mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait} <: Template end """ - FakeTemplate{PointEstimate, SingleOutput}() + FakeTemplate{PointEstimate, SingleOutput, PointInject}() A [`Template`](@ref) whose [`Model`](@ref) will predict 0 for each observation. """ @@ -119,7 +119,7 @@ function StatsBase.fit( return FakeModel{E, O}(template.predictor, num_variates) end -StatsBase.predict(m::FakeModel, inputs) = m.predictor(m.num_variates, inputs) +StatsBase.predict(m::FakeModel, inputs::AbstractMatrix) = m.predictor(m.num_variates, inputs) """ test_interface(template::Template; inputs=rand(5, 5), outputs=rand(5, 5)) @@ -129,23 +129,28 @@ Can be used as an initial test to verify the API has been correctly implemented. """ function test_interface(template::Template; kwargs...) @testset "Models API Interface Test: $(nameof(typeof(template)))" begin - test_interface(template, estimate_type(template), output_type(template); kwargs...) + test_interface( + template, + estimate_type(template), + output_type(template), + inject_type(template); + kwargs... + ) end end function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}; + template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointInject}; inputs=rand(5, 5), outputs=rand(1, 5), ) predictions = test_common(template, inputs, outputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} @test size(predictions) == size(outputs) @test size(predictions, 1) == 1 end function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}; + template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointInject}; inputs=rand(5, 5), outputs=rand(2, 5), ) predictions = test_common(template, inputs, outputs) @@ -154,7 +159,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}; + template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointInject}; inputs=rand(5, 5), outputs=rand(1, 5), ) predictions = test_common(template, inputs, outputs) @@ -164,7 +169,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}; + template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointInject}; inputs=rand(5, 5), outputs=rand(3, 5) ) predictions = test_common(template, inputs, outputs) @@ -210,6 +215,7 @@ function test_common(template, inputs, outputs) @testset "traits" begin @test estimate_type(template) == estimate_type(model) @test output_type(template) == output_type(model) + @test inject_type(template) == inject_type(model) end @testset "submodels" begin diff --git a/test/test_utils.jl b/test/test_utils.jl index 90efa0e..7197cf2 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -20,4 +20,9 @@ test_interface(temp) end + @testset "FakeTemplate{PointEstimate, SingleOutput} with Vector{<:Vector}" begin + temp = FakeTemplate{PointEstimate, SingleOutput}() + test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) + end + end From 38f8c1e4025e8fb3cfce16b102ac51b8a3a038f7 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Feb 2021 11:29:20 +0000 Subject: [PATCH 06/26] Add testutil for DistributionInject --- src/test_utils.jl | 70 ++++++++++++++++++++++++++++++++-------------- test/test_utils.jl | 24 +++++++++------- 2 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index dab1345..3101679 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -16,7 +16,7 @@ export FakeModel, FakeTemplate export test_interface """ - FakeTemplate{E <: EstimateTrait, O <: OutputTrait} <: Template + FakeTemplate{E <: EstimateTrait, O <: OutputTrait, I <: InjectTrait} <: Template This template is a [test double](https://en.wikipedia.org/wiki/Test_double) for testing purposes. It should be defined (before fitting) with a `predictor`, which can be changed by @@ -31,7 +31,7 @@ mutating the field. - [`fit`](@ref) does not learn anything, it just creates an instance of the corresponding [`Model`](@ref). - [`predict`](@ref) applies the `predictor` to the inputs. """ -mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait} <: Template +mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait, I<:InjectTrait} <: Template predictor::Function end @@ -40,8 +40,8 @@ end A [`Template`](@ref) whose [`Model`](@ref) will predict 0 for each observation. """ -function FakeTemplate{PointEstimate, SingleOutput}() - FakeTemplate{PointEstimate, SingleOutput}() do num_variates, inputs +function FakeTemplate{PointEstimate, SingleOutput, PointInject}() + FakeTemplate{PointEstimate, SingleOutput, PointInject}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") inputs = NamedDimsArray{(:features, :observations)}(inputs) return NamedDimsArray{(:variates, :observations)}( @@ -51,13 +51,13 @@ function FakeTemplate{PointEstimate, SingleOutput}() end """ - FakeTemplate{PointEstimate, MultiOutput}() + FakeTemplate{PointEstimate, MultiOutput, PointInject}() A [`Template`](@ref) whose [`Model`](@ref) will predict a vector of 0s for each observation. The input and output will have the same dimension. """ -function FakeTemplate{PointEstimate, MultiOutput}() - FakeTemplate{PointEstimate, MultiOutput}() do num_variates, inputs +function FakeTemplate{PointEstimate, MultiOutput, PointInject}() + FakeTemplate{PointEstimate, MultiOutput, PointInject}() do num_variates, inputs inputs = NamedDimsArray{(:features, :observations)}(inputs) return NamedDimsArray{(:variates, :observations)}( zeros(num_variates, size(inputs, :observations)) @@ -66,13 +66,13 @@ function FakeTemplate{PointEstimate, MultiOutput}() end """ - FakeTemplate{DistributionEstimate, SingleOutput}() + FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() A [`Template`](@ref) whose [`Model`](@ref) will predict a univariate normal distribution (with zero mean and unit standard deviation) for each observation. """ -function FakeTemplate{DistributionEstimate, SingleOutput}() - FakeTemplate{DistributionEstimate, SingleOutput}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() + FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") inputs = NamedDimsArray{(:features, :observations)}(inputs) return NoncentralT.(3.0, zeros(size(inputs, :observations))) @@ -80,46 +80,63 @@ function FakeTemplate{DistributionEstimate, SingleOutput}() end """ - FakeTemplate{DistributionEstimate, MultiOutput}() + FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal distribution (with zero-vector mean and identity covariance matrix) for each observation. """ -function FakeTemplate{DistributionEstimate, MultiOutput}() - FakeTemplate{DistributionEstimate, MultiOutput}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() + FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() do num_variates, inputs std_dev = ones(num_variates) return [Product(Normal.(0, std_dev)) for _ in 1:size(inputs, 2)] end end +""" + FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() + +A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal +distribution (with zero-vector mean and identity covariance matrix) for each observation. +""" +function FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() + FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() do num_variates, inputs + std_dev = ones(num_variates) + means = mean.(inputs) + return [Product(Normal.(m, std_dev)) for m in means] + end +end + """ FakeModel A fake Model for testing purposes. See [`FakeTemplate`](@ref) for details. """ -mutable struct FakeModel{E<:EstimateTrait, O<:OutputTrait} <: Model +mutable struct FakeModel{E<:EstimateTrait, O<:OutputTrait, I<:InjectTrait} <: Model predictor::Function num_variates::Int end -Models.estimate_type(::Type{<:FakeModel{E, O}}) where {E, O} = E -Models.output_type(::Type{<:FakeModel{E, O}}) where {E, O} = O +Models.estimate_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = E +Models.output_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = O +Models.inject_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = I -Models.estimate_type(::Type{<:FakeTemplate{E, O}}) where {E, O} = E -Models.output_type(::Type{<:FakeTemplate{E, O}}) where {E, O} = O +Models.estimate_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = E +Models.output_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = O +Models.inject_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = I function StatsBase.fit( - template::FakeTemplate{E, O}, + template::FakeTemplate{E, O, I}, outputs, inputs, weights=uweights(Float32, size(outputs, 2)) -) where {E, O} +) where {E, O, I} outputs = NamedDimsArray{(:variates, :observations)}(outputs) num_variates = size(outputs, :variates) - return FakeModel{E, O}(template.predictor, num_variates) + return FakeModel{E, O, I}(template.predictor, num_variates) end StatsBase.predict(m::FakeModel, inputs::AbstractMatrix) = m.predictor(m.num_variates, inputs) +StatsBase.predict(m::FakeModel, inputs::AbstractVector{<:Normal}) = m.predictor(m.num_variates, inputs) """ test_interface(template::Template; inputs=rand(5, 5), outputs=rand(5, 5)) @@ -178,6 +195,17 @@ function test_interface( @test all(length.(predictions) .== size(outputs, 1)) end +function test_interface( + template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{DistributionInject}; + inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5) +) + predictions = test_common(template, inputs, outputs) + @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} + @test length(predictions) == size(outputs, 2) + @test all(length.(predictions) .== size(outputs, 1)) + @test mean.(mean.(predictions)) == [i for i in 1:5] +end + function test_names(template, model) template_type_name = string(nameof(typeof(template))) template_base_name_match = match(r"(.*)Template", template_type_name) diff --git a/test/test_utils.jl b/test/test_utils.jl index 7197cf2..d981761 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -1,28 +1,32 @@ @testset "test_utils.jl" begin - @testset "FakeTemplate{PointEstimate, SingleOutput}" begin - temp = FakeTemplate{PointEstimate, SingleOutput}() + @testset "FakeTemplate{PointEstimate, SingleOutput, PointInject}" begin + temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() test_interface(temp) end - @testset "FakeTemplate{PointEstimate, MultiOutput}" begin - temp = FakeTemplate{PointEstimate, MultiOutput}() + @testset "FakeTemplate{PointEstimate, MultiOutput, PointInject}" begin + temp = FakeTemplate{PointEstimate, MultiOutput, PointInject}() test_interface(temp) end - @testset "FakeTemplate{DistributionEstimate, SingleOutput}" begin - temp = FakeTemplate{DistributionEstimate, SingleOutput}() + @testset "FakeTemplate{DistributionEstimate, SingleOutput, PointInject}" begin + temp = FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() test_interface(temp) end - @testset "FakeTemplate{DistributionEstimate, MultiOutput}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput}() + @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointInject}" begin + temp = FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() test_interface(temp) end - @testset "FakeTemplate{PointEstimate, SingleOutput} with Vector{<:Vector}" begin - temp = FakeTemplate{PointEstimate, SingleOutput}() + @testset "FakeTemplate{PointEstimate, SingleOutput, PointInject} with Vector{<:Vector}" begin + temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) end + @testset "FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}" begin + temp = FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() + test_interface(temp) + end end From e14cc3f70d4da27b8a6ed1fbaf2404d25bca36c0 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Feb 2021 15:11:06 +0000 Subject: [PATCH 07/26] Tests for PointOrDistributionInject --- src/test_utils.jl | 28 ++++++++++++++++++++++++++++ test/test_utils.jl | 17 ++++++++++++----- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 3101679..abe5110 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -106,6 +106,23 @@ function FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() end end +""" + FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() + +A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal +distribution (with zero-vector mean and identity covariance matrix) for each observation. +""" +function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() + FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() do num_variates, inputs + std_dev = ones(num_variates) + means = _handle_inputs(inputs) + return [Product(Normal.(m, std_dev)) for m in means] + end +end + +_handle_inputs(inputs::AbstractVector{<:Normal}) = mean.(inputs) +_handle_inputs(inputs::AbstractMatrix) = only([mean(inputs[:, m]) for m in 1:size(inputs, 2)]) + """ FakeModel @@ -206,6 +223,17 @@ function test_interface( @test mean.(mean.(predictions)) == [i for i in 1:5] end +function test_interface( + template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionInject}; + inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5) +) + predictions = test_common(template, inputs, outputs) + @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} + @test length(predictions) == size(outputs, 2) + @test all(length.(predictions) .== size(outputs, 1)) + @test mean.(mean.(predictions)) == [i for i in 1:5] +end + function test_names(template, model) template_type_name = string(nameof(typeof(template))) template_base_name_match = match(r"(.*)Template", template_type_name) diff --git a/test/test_utils.jl b/test/test_utils.jl index d981761..eb68649 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -3,6 +3,9 @@ @testset "FakeTemplate{PointEstimate, SingleOutput, PointInject}" begin temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() test_interface(temp) + + temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() + test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) end @testset "FakeTemplate{PointEstimate, MultiOutput, PointInject}" begin @@ -20,13 +23,17 @@ test_interface(temp) end - @testset "FakeTemplate{PointEstimate, SingleOutput, PointInject} with Vector{<:Vector}" begin - temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() - test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) - end - @testset "FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}" begin temp = FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() test_interface(temp) end + + @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}" begin + temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() + test_interface(temp) + + temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() + test_interface(temp; inputs=hcat([[i for i in 1:5] for j in 1:5]), outputs=rand(3, 5)) + end + end From 1bc6c21a6a151e22d7dea88cfcbca3fe3f362670 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Wed, 17 Feb 2021 17:35:24 +0000 Subject: [PATCH 08/26] Switch up default kwargs to work with external models --- src/test_utils.jl | 6 ++---- test/test_utils.jl | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index abe5110..7de23eb 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -121,7 +121,7 @@ function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInje end _handle_inputs(inputs::AbstractVector{<:Normal}) = mean.(inputs) -_handle_inputs(inputs::AbstractMatrix) = only([mean(inputs[:, m]) for m in 1:size(inputs, 2)]) +_handle_inputs(inputs::AbstractMatrix) = [mean(inputs[m, :]) for m in 1:size(inputs, 2)] """ FakeModel @@ -220,18 +220,16 @@ function test_interface( @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} @test length(predictions) == size(outputs, 2) @test all(length.(predictions) .== size(outputs, 1)) - @test mean.(mean.(predictions)) == [i for i in 1:5] end function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionInject}; - inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5) + inputs=hcat([[i for i in 1:5] for j in 1:5]...), outputs=rand(3, 5) ) predictions = test_common(template, inputs, outputs) @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} @test length(predictions) == size(outputs, 2) @test all(length.(predictions) .== size(outputs, 1)) - @test mean.(mean.(predictions)) == [i for i in 1:5] end function test_names(template, model) diff --git a/test/test_utils.jl b/test/test_utils.jl index eb68649..b450bd4 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -30,10 +30,10 @@ @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}" begin temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() - test_interface(temp) + test_interface(temp; inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5)) temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() - test_interface(temp; inputs=hcat([[i for i in 1:5] for j in 1:5]), outputs=rand(3, 5)) + test_interface(temp) end end From c8923d21cc42a74659eb759dfab600b7c2c7c9fd Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 18 Feb 2021 11:52:56 +0000 Subject: [PATCH 09/26] Expand test interface for PointOrDistInject case to test forboth predict methods --- src/test_utils.jl | 18 +++++++++++++----- src/traits.jl | 3 ++- test/test_utils.jl | 3 --- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 7de23eb..275186a 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -95,8 +95,9 @@ end """ FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() -A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal -distribution (with zero-vector mean and identity covariance matrix) for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept a vector of distributions to predict +a multivariate normal distribution (with means matching those of the passed distributions +and identity covariance matrix) for each observation. """ function FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() do num_variates, inputs @@ -109,8 +110,9 @@ end """ FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() -A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal -distribution (with zero-vector mean and identity covariance matrix) for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict a multivariate normal distribution (with means matching those of the passed +observations and identity covariance matrix) for each observation. """ function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() do num_variates, inputs @@ -224,12 +226,18 @@ end function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionInject}; - inputs=hcat([[i for i in 1:5] for j in 1:5]...), outputs=rand(3, 5) + inputs=rand(5, 5), outputs=rand(3, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], ) predictions = test_common(template, inputs, outputs) @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} @test length(predictions) == size(outputs, 2) @test all(length.(predictions) .== size(outputs, 1)) + + model = fit(template, outputs, inputs) + predictions = predict(model, distribution_inputs) + @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} + @test length(predictions) == size(outputs, 2) + @test all(length.(predictions) .== size(outputs, 1)) end function test_names(template, model) diff --git a/src/traits.jl b/src/traits.jl index 6e15a01..36aeb9d 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -63,7 +63,8 @@ output_type(T::Type) = throw(MethodError(output_type, (T,))) # to prevent recur InjectTrait The `InjectTrait` specifies if the model supports point or distribution injections to predict, -denoted by [`PointInject`](@ref) or [`DistributionInject`](@ref), respectively. +denoted by [`PointInject`](@ref) or [`DistributionInject`](@ref), respectively. A model can +also be implemented in such a way as to allow [`PointOrDistributionInject`](@ref). """ abstract type InjectTrait end diff --git a/test/test_utils.jl b/test/test_utils.jl index b450bd4..fab5880 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -29,9 +29,6 @@ end @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() - test_interface(temp; inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5)) - temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() test_interface(temp) end From be30c60b4c9098cb6e347812d530ceebafc079d1 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 18 Feb 2021 17:24:35 +0000 Subject: [PATCH 10/26] Chance required dist type to Sampleable --- src/test_utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 275186a..b8c751b 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -122,7 +122,7 @@ function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInje end end -_handle_inputs(inputs::AbstractVector{<:Normal}) = mean.(inputs) +_handle_inputs(inputs::AbstractVector{<:Sampleable}) = mean.(inputs) _handle_inputs(inputs::AbstractMatrix) = [mean(inputs[m, :]) for m in 1:size(inputs, 2)] """ @@ -155,7 +155,7 @@ function StatsBase.fit( end StatsBase.predict(m::FakeModel, inputs::AbstractMatrix) = m.predictor(m.num_variates, inputs) -StatsBase.predict(m::FakeModel, inputs::AbstractVector{<:Normal}) = m.predictor(m.num_variates, inputs) +StatsBase.predict(m::FakeModel, inputs::AbstractVector{<:Sampleable}) = m.predictor(m.num_variates, inputs) """ test_interface(template::Template; inputs=rand(5, 5), outputs=rand(5, 5)) From aa40deb46a88203e99a79e34eb8b7db56202e289 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Thu, 18 Feb 2021 17:48:18 +0000 Subject: [PATCH 11/26] Renaming to PredictInputTrait --- src/Models.jl | 7 ++++-- src/test_utils.jl | 62 +++++++++++++++++++++++----------------------- src/traits.jl | 34 ++++++++++++------------- test/test_utils.jl | 26 +++++++++---------- test/traits.jl | 14 +++++------ 5 files changed, 73 insertions(+), 70 deletions(-) diff --git a/src/Models.jl b/src/Models.jl index 2baded0..20a34b0 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -3,10 +3,13 @@ module Models import StatsBase: fit, predict export Model, Template -export fit, predict, submodels, estimate_type, output_type, inject_type +export fit, predict, submodels, estimate_type, output_type, predict_input_type export EstimateTrait, PointEstimate, DistributionEstimate export OutputTrait, SingleOutput, MultiOutput -export InjectTrait, PointInject, DistributionInject, PointOrDistributionInject +export PredictInputTrait, + PointPredictInput, + DistributionPredictInput, + PointOrDistributionPredictInput """ Template diff --git a/src/test_utils.jl b/src/test_utils.jl index b8c751b..d3d34d8 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -16,7 +16,7 @@ export FakeModel, FakeTemplate export test_interface """ - FakeTemplate{E <: EstimateTrait, O <: OutputTrait, I <: InjectTrait} <: Template + FakeTemplate{E <: EstimateTrait, O <: OutputTrait, I <: PredictInputTrait} <: Template This template is a [test double](https://en.wikipedia.org/wiki/Test_double) for testing purposes. It should be defined (before fitting) with a `predictor`, which can be changed by @@ -31,17 +31,17 @@ mutating the field. - [`fit`](@ref) does not learn anything, it just creates an instance of the corresponding [`Model`](@ref). - [`predict`](@ref) applies the `predictor` to the inputs. """ -mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait, I<:InjectTrait} <: Template +mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait, I<:PredictInputTrait} <: Template predictor::Function end """ - FakeTemplate{PointEstimate, SingleOutput, PointInject}() + FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will predict 0 for each observation. """ -function FakeTemplate{PointEstimate, SingleOutput, PointInject}() - FakeTemplate{PointEstimate, SingleOutput, PointInject}() do num_variates, inputs +function FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() + FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") inputs = NamedDimsArray{(:features, :observations)}(inputs) return NamedDimsArray{(:variates, :observations)}( @@ -51,13 +51,13 @@ function FakeTemplate{PointEstimate, SingleOutput, PointInject}() end """ - FakeTemplate{PointEstimate, MultiOutput, PointInject}() + FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will predict a vector of 0s for each observation. The input and output will have the same dimension. """ -function FakeTemplate{PointEstimate, MultiOutput, PointInject}() - FakeTemplate{PointEstimate, MultiOutput, PointInject}() do num_variates, inputs +function FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() + FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() do num_variates, inputs inputs = NamedDimsArray{(:features, :observations)}(inputs) return NamedDimsArray{(:variates, :observations)}( zeros(num_variates, size(inputs, :observations)) @@ -66,13 +66,13 @@ function FakeTemplate{PointEstimate, MultiOutput, PointInject}() end """ - FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() + FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will predict a univariate normal distribution (with zero mean and unit standard deviation) for each observation. """ -function FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() - FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() + FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") inputs = NamedDimsArray{(:features, :observations)}(inputs) return NoncentralT.(3.0, zeros(size(inputs, :observations))) @@ -80,27 +80,27 @@ function FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() end """ - FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() + FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal distribution (with zero-vector mean and identity covariance matrix) for each observation. """ -function FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() - FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() + FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() do num_variates, inputs std_dev = ones(num_variates) return [Product(Normal.(0, std_dev)) for _ in 1:size(inputs, 2)] end end """ - FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() + FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will accept a vector of distributions to predict a multivariate normal distribution (with means matching those of the passed distributions and identity covariance matrix) for each observation. """ -function FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() - FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() + FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs std_dev = ones(num_variates) means = mean.(inputs) return [Product(Normal.(m, std_dev)) for m in means] @@ -108,14 +108,14 @@ function FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() end """ - FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() + FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to predict a multivariate normal distribution (with means matching those of the passed observations and identity covariance matrix) for each observation. """ -function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() - FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() + FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() do num_variates, inputs std_dev = ones(num_variates) means = _handle_inputs(inputs) return [Product(Normal.(m, std_dev)) for m in means] @@ -130,18 +130,18 @@ _handle_inputs(inputs::AbstractMatrix) = [mean(inputs[m, :]) for m in 1:size(inp A fake Model for testing purposes. See [`FakeTemplate`](@ref) for details. """ -mutable struct FakeModel{E<:EstimateTrait, O<:OutputTrait, I<:InjectTrait} <: Model +mutable struct FakeModel{E<:EstimateTrait, O<:OutputTrait, I<:PredictInputTrait} <: Model predictor::Function num_variates::Int end Models.estimate_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = E Models.output_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = O -Models.inject_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = I +Models.predict_input_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = I Models.estimate_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = E Models.output_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = O -Models.inject_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = I +Models.predict_input_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = I function StatsBase.fit( template::FakeTemplate{E, O, I}, @@ -169,14 +169,14 @@ function test_interface(template::Template; kwargs...) template, estimate_type(template), output_type(template), - inject_type(template); + predict_input_type(template); kwargs... ) end end function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointInject}; + template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointPredictInput}; inputs=rand(5, 5), outputs=rand(1, 5), ) predictions = test_common(template, inputs, outputs) @@ -186,7 +186,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointInject}; + template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointPredictInput}; inputs=rand(5, 5), outputs=rand(2, 5), ) predictions = test_common(template, inputs, outputs) @@ -195,7 +195,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointInject}; + template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointPredictInput}; inputs=rand(5, 5), outputs=rand(1, 5), ) predictions = test_common(template, inputs, outputs) @@ -205,7 +205,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointInject}; + template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointPredictInput}; inputs=rand(5, 5), outputs=rand(3, 5) ) predictions = test_common(template, inputs, outputs) @@ -215,7 +215,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{DistributionInject}; + template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5) ) predictions = test_common(template, inputs, outputs) @@ -225,7 +225,7 @@ function test_interface( end function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionInject}; + template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; inputs=rand(5, 5), outputs=rand(3, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], ) predictions = test_common(template, inputs, outputs) @@ -277,7 +277,7 @@ function test_common(template, inputs, outputs) @testset "traits" begin @test estimate_type(template) == estimate_type(model) @test output_type(template) == output_type(model) - @test inject_type(template) == inject_type(model) + @test predict_input_type(template) == predict_input_type(model) end @testset "submodels" begin diff --git a/src/traits.jl b/src/traits.jl index 36aeb9d..4554d5e 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -60,43 +60,43 @@ output_type(::T) where T = output_type(T) output_type(T::Type) = throw(MethodError(output_type, (T,))) # to prevent recursion """ - InjectTrait + PredictInputTrait -The `InjectTrait` specifies if the model supports point or distribution injections to predict, -denoted by [`PointInject`](@ref) or [`DistributionInject`](@ref), respectively. A model can -also be implemented in such a way as to allow [`PointOrDistributionInject`](@ref). +The `PredictInputTrait` specifies if the model supports point or distribution injections to predict, +denoted by [`PointPredictInput`](@ref) or [`DistributionPredictInput`](@ref), respectively. A model can +also be implemented in such a way as to allow [`PointOrDistributionPredictInput`](@ref). """ -abstract type InjectTrait end +abstract type PredictInputTrait end """ - PointInject <: InjectTrait + PointPredictInput <: PredictInputTrait Specifies that the [`Model`](@ref) accepts real-valued input variables to `predict`. """ -abstract type PointInject <: InjectTrait end +abstract type PointPredictInput <: PredictInputTrait end """ - DistributionInject <: InjectTrait + DistributionPredictInput <: PredictInputTrait Specifies that the [`Model`](@ref) accepts a distribution over the input variables to `predict`. """ -abstract type DistributionInject <: InjectTrait end +abstract type DistributionPredictInput <: PredictInputTrait end """ - PointOrDistributionInject <: InjectTrait + PointOrDistributionPredictInput <: PredictInputTrait Specifies that the [`Model`](@ref) accepts real-values or a distribution over the input variables to `predict`. """ -abstract type PointOrDistributionInject <: InjectTrait end +abstract type PointOrDistributionPredictInput <: PredictInputTrait end """ - inject_type(::T) where T = inject_type(T) + predict_input_type(::T) where T = predict_input_type(T) -Return the [`InjectTrait`] of the [`Model`](@ref) or [`Template`](@ref). +Return the [`PredictInputTrait`] of the [`Model`](@ref) or [`Template`](@ref). """ -inject_type(::T) where T = inject_type(T) -inject_type(T::Type) = throw(MethodError(inject_type, (T,))) # to prevent recursion +predict_input_type(::T) where T = predict_input_type(T) +predict_input_type(T::Type) = throw(MethodError(predict_input, (T,))) # to prevent recursion -inject_type(::Type{<:Model}) = PointInject -inject_type(::Type{<:Template}) = PointInject +predict_input_type(::Type{<:Model}) = PointPredictInput +predict_input_type(::Type{<:Template}) = PointPredictInput diff --git a/test/test_utils.jl b/test/test_utils.jl index fab5880..0572f2b 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -1,35 +1,35 @@ @testset "test_utils.jl" begin - @testset "FakeTemplate{PointEstimate, SingleOutput, PointInject}" begin - temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() + @testset "FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}" begin + temp = FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() test_interface(temp) - temp = FakeTemplate{PointEstimate, SingleOutput, PointInject}() + temp = FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) end - @testset "FakeTemplate{PointEstimate, MultiOutput, PointInject}" begin - temp = FakeTemplate{PointEstimate, MultiOutput, PointInject}() + @testset "FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}" begin + temp = FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() test_interface(temp) end - @testset "FakeTemplate{DistributionEstimate, SingleOutput, PointInject}" begin - temp = FakeTemplate{DistributionEstimate, SingleOutput, PointInject}() + @testset "FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}" begin + temp = FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() test_interface(temp) end - @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointInject}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, PointInject}() + @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}" begin + temp = FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() test_interface(temp) end - @testset "FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, DistributionInject}() + @testset "FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}" begin + temp = FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() test_interface(temp) end - @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionInject}() + @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}" begin + temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() test_interface(temp) end diff --git a/test/traits.jl b/test/traits.jl index c8662dd..cda9681 100644 --- a/test/traits.jl +++ b/test/traits.jl @@ -29,19 +29,19 @@ struct DummyModel <: Model end end - @testset "Inject Trait" begin + @testset "PredictInput Trait" begin @testset "Default" begin - @test inject_type(DummyTemplate) == inject_type(DummyModel) == PointInject + @test predict_input_type(DummyTemplate) == predict_input_type(DummyModel) == PointPredictInput end - injects = (DistributionInject, PointOrDistributionInject) + predict_input = (DistributionPredictInput, PointOrDistributionPredictInput) - @testset "$inj is defined" for inj in injects - inject_type(m::Type{<:DummyTemplate}) = inj - inject_type(m::Type{<:DummyModel}) = inj + @testset "$pinputs is defined" for pinputs in predict_input + predict_input_type(m::Type{<:DummyTemplate}) = pinputs + predict_input_type(m::Type{<:DummyModel}) = pinputs - @test inject_type(DummyTemplate) == inject_type(DummyModel) == inj + @test predict_input_type(DummyTemplate) == predict_input_type(DummyModel) == pinputs end end From 0ed9fd5ac1af3bbc4f4c78bbe204b2293e7aaf8b Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Fri, 19 Feb 2021 10:01:51 +0000 Subject: [PATCH 12/26] Add docs for distribution predict input test interface --- docs/src/api.md | 4 ++++ docs/src/design.md | 5 +++++ docs/src/testutils.md | 17 +++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/docs/src/api.md b/docs/src/api.md index 054ad91..6c1b3f5 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -23,4 +23,8 @@ DistributionEstimate OutputTrait SingleOutput MultiOutput +PredictInputTrait +PointPredictInput +DistributionPredictInput +PointOrDistributionPredictInput ``` diff --git a/docs/src/design.md b/docs/src/design.md index f976c4d..4f348a2 100644 --- a/docs/src/design.md +++ b/docs/src/design.md @@ -72,9 +72,14 @@ Here are the current [`Model`](@ref) traits in use and their possible values: - [`output_type`](@ref) - determines how many output variates a [`Model`](@ref) can learn - [`SingleOutput`](@ref): Fits and predicts on a single output only. - [`MultiOutput`](@ref): Fits and predicts on multiple outputs at a time. + - [`predict_input_type`](@ref) - determines which datatypes a [`Model`](@ref) can accept at predict time. + - [`PointPredictInput`](@ref): Real valued input variables accepted at predict time. + - [`DistributionPredictInput`](@ref): Distributions over input variables accepted at predict time. + - [`PointOrDistributionPredictInput`](@ref): Either real valued or distributions of input variables accepted at predict time. The traits always agree between the [`Model`](@ref) and the [`Template`](@ref). Every [`Model`](@ref) and [`Template`](@ref) should define all the listed traits. +If left undefined, the ['PredictInputTrait'](@ref) will have the default value of [`PointPredictInput`](@ref). This package uses traits implemented such that the trait function returns an `abstract type` (rather than an instance). That means to check a trait one uses: diff --git a/docs/src/testutils.md b/docs/src/testutils.md index f33e835..faf1dd9 100644 --- a/docs/src/testutils.md +++ b/docs/src/testutils.md @@ -9,6 +9,23 @@ TestUtils test_interface ``` +### Note on PredictInputTrait Interface Tests + +In the case where the [`PredictInputTrait`](@ref) is [`DistributionPredictInput`](@ref) or [`PointOrDistributionPredictInput`](@ref) the the Models API requires only that the distribution in question is `Sampleable`. +When using `Models.TestUtils.test_interface` to test a model where distributions can be passed to `predict`, the user should provide `inputs` of the distribution type appropriate to their model. +In the example below the `CustomModel` accepts `MvNormal` distributions to `predict`. + +```julia +using CustomModels +using Distributions +using Models.TestUtils + +test_interface( + CustomModelTemplate(); + distribution_inputs=[MvNormal(5, 1) for _ in 1:5], +) +``` + ## Test Fakes ```@autodocs Modules = [Models.TestUtils] From 6b59b4e6a46917ddbc038b36fccc1f9e8775fd38 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Fri, 19 Feb 2021 12:23:56 +0000 Subject: [PATCH 13/26] Add all trait combinations to test utils --- src/test_utils.jl | 196 ++++++++++++++++++++++++++++++++++++++++++--- test/test_utils.jl | 31 ++++++- 2 files changed, 216 insertions(+), 11 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index d3d34d8..47efae9 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -38,7 +38,8 @@ end """ FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will predict 0 for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict 0 +for each observation. """ function FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() do num_variates, inputs @@ -53,8 +54,8 @@ end """ FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will predict a vector of 0s for each observation. -The input and output will have the same dimension. +A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a +vector of 0s for each observation. The input and output will have the same dimension. """ function FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() do num_variates, inputs @@ -68,8 +69,9 @@ end """ FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will predict a univariate normal -distribution (with zero mean and unit standard deviation) for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a +univariate normal distribution (with zero mean and unit standard deviation) for each +observation. """ function FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() do num_variates, inputs @@ -82,8 +84,9 @@ end """ FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will predict a multivariate normal -distribution (with zero-vector mean and identity covariance matrix) for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a +multivariate normal distribution (with zero-vector mean and identity covariance matrix) for +each observation. """ function FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() do num_variates, inputs @@ -92,12 +95,61 @@ function FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() end end +""" + FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() + +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict 0 for +each observation. +""" +function FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() + FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() do num_variates, inputs + @assert(num_variates == 1, "$num_variates != 1") + samples = hcat([mean.(inputs) for _ in 1:5]...) + inputs = NamedDimsArray{(:features, :observations)}(samples) + return NamedDimsArray{(:variates, :observations)}( + zeros(1, size(inputs, :observations)) + ) + end +end + +""" + FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() + +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict a +vector of 0s for each observation. The input and output will have the same dimension. +""" +function FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() + FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs + samples = hcat([mean.(inputs) for _ in 1:5]...) + inputs = NamedDimsArray{(:features, :observations)}(samples) + return NamedDimsArray{(:variates, :observations)}( + zeros(num_variates, size(inputs, :observations)) + ) + end +end + +""" + FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() + +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict a +univariate normal distribution (with zero mean and unit standard deviation) for each +observation. +""" +function FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() + FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() do num_variates, inputs + @assert(num_variates == 1, "$num_variates != 1") + samples = hcat([mean.(inputs) for _ in 1:5]...) + inputs = NamedDimsArray{(:features, :observations)}(samples) + return NoncentralT.(3.0, zeros(size(inputs, :observations))) + end +end + """ FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept a vector of distributions to predict -a multivariate normal distribution (with means matching those of the passed distributions -and identity covariance matrix) for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict a +multivariate normal distribution (with means matching those of the passed distributions and +identity covariance matrix) for each observation. """ function FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs @@ -107,6 +159,55 @@ function FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInpu end end +""" + FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() + +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict 0 for each observation. +""" +function FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() + FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() do num_variates, inputs + @assert(num_variates == 1, "$num_variates != 1") + samples = hcat([_handle_inputs(inputs) for _ in 1:5]...) + inputs = NamedDimsArray{(:features, :observations)}(samples) + return NamedDimsArray{(:variates, :observations)}( + zeros(1, size(inputs, :observations)) + ) + end +end + +""" + FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() + +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict a vector of 0s for each observation. The input and output will have the same dimension. +""" +function FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() + FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() do num_variates, inputs + samples = hcat([_handle_inputs(inputs) for _ in 1:5]...) + inputs = NamedDimsArray{(:features, :observations)}(samples) + return NamedDimsArray{(:variates, :observations)}( + zeros(num_variates, size(inputs, :observations)) + ) + end +end + +""" + FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() + +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict a univariate normal distribution (with zero mean and unit standard deviation) for +each observation. +""" +function FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() + FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() do num_variates, inputs + @assert(num_variates == 1, "$num_variates != 1") + samples = hcat([_handle_inputs(inputs) for _ in 1:5]...) + inputs = NamedDimsArray{(:features, :observations)}(samples) + return NoncentralT.(3.0, zeros(size(inputs, :observations))) + end +end + """ FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() @@ -214,6 +315,35 @@ function test_interface( @test all(length.(predictions) .== size(outputs, 1)) end +function test_interface( + template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{DistributionPredictInput}; + inputs=[Normal(m, 1) for m in 1:5], outputs=rand(1, 5), +) + predictions = test_common(template, inputs, outputs) + @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} + @test size(predictions) == size(outputs) + @test size(predictions, 1) == 1 +end + +function test_interface( + template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; + inputs=[Normal(m, 1) for m in 1:5], outputs=rand(2, 5), +) + predictions = test_common(template, inputs, outputs) + @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} + @test size(predictions) == size(outputs) +end + +function test_interface( + template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{DistributionPredictInput}; + inputs=[Normal(m, 1) for m in 1:5], outputs=rand(1, 5), +) + predictions = test_common(template, inputs, outputs) + @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} + @test length(predictions) == size(outputs, 2) + @test all(length.(predictions) .== size(outputs, 1)) +end + function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5) @@ -224,6 +354,52 @@ function test_interface( @test all(length.(predictions) .== size(outputs, 1)) end +function test_interface( + template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointOrDistributionPredictInput}; + inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], +) + predictions = test_common(template, inputs, outputs) + @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} + @test size(predictions) == size(outputs) + @test size(predictions, 1) == 1 + + model = fit(template, outputs, inputs) + predictions = predict(model, distribution_inputs) + @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} + @test size(predictions) == size(outputs) + @test size(predictions, 1) == 1 +end + +function test_interface( + template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; + inputs=rand(5, 5), outputs=rand(2, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], +) + predictions = test_common(template, inputs, outputs) + @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} + @test size(predictions) == size(outputs) + + model = fit(template, outputs, inputs) + predictions = predict(model, distribution_inputs) + @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} + @test size(predictions) == size(outputs) +end + +function test_interface( + template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointOrDistributionPredictInput}; + inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], +) + predictions = test_common(template, inputs, outputs) + @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} + @test length(predictions) == size(outputs, 2) + @test all(length.(predictions) .== size(outputs, 1)) + + model = fit(template, outputs, inputs) + predictions = predict(model, distribution_inputs) + @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} + @test length(predictions) == size(outputs, 2) + @test all(length.(predictions) .== size(outputs, 1)) +end + function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; inputs=rand(5, 5), outputs=rand(3, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], diff --git a/test/test_utils.jl b/test/test_utils.jl index 0572f2b..1a29498 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -23,14 +23,43 @@ test_interface(temp) end + @testset "FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}" begin + temp = FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() + test_interface(temp) + end + + @testset "FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}" begin + temp = FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() + test_interface(temp) + end + + @testset "FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}" begin + temp = FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() + test_interface(temp) + end + @testset "FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}" begin temp = FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() test_interface(temp) end + @testset "FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}" begin + temp = FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() + test_interface(temp) + end + + @testset "FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}" begin + temp = FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() + test_interface(temp) + end + + @testset "FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}" begin + temp = FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() + test_interface(temp) + end + @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}" begin temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() test_interface(temp) end - end From a82796e5443caee7ebb5a70a46b7ecfa9728199a Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Feb 2021 14:50:57 +0000 Subject: [PATCH 14/26] Change dist dims of predict inputs for distribution cases --- src/test_utils.jl | 82 +++++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 47efae9..93f7a11 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -98,14 +98,14 @@ end """ FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict 0 for -each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and +return 0 for each observation. """ function FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") - samples = hcat([mean.(inputs) for _ in 1:5]...) - inputs = NamedDimsArray{(:features, :observations)}(samples) + means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) + inputs = NamedDimsArray{(:features, :observations)}(means) return NamedDimsArray{(:variates, :observations)}( zeros(1, size(inputs, :observations)) ) @@ -115,13 +115,13 @@ end """ FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict a -vector of 0s for each observation. The input and output will have the same dimension. +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and +return a vector of 0s for each observation. The input and output will have the same dimension. """ function FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs - samples = hcat([mean.(inputs) for _ in 1:5]...) - inputs = NamedDimsArray{(:features, :observations)}(samples) + means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) + inputs = NamedDimsArray{(:features, :observations)}(means) return NamedDimsArray{(:variates, :observations)}( zeros(num_variates, size(inputs, :observations)) ) @@ -131,15 +131,15 @@ end """ FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict a -univariate normal distribution (with zero mean and unit standard deviation) for each +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and +return a univariate normal distribution (with zero mean and unit standard deviation) for each observation. """ function FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") - samples = hcat([mean.(inputs) for _ in 1:5]...) - inputs = NamedDimsArray{(:features, :observations)}(samples) + means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) + inputs = NamedDimsArray{(:features, :observations)}(means) return NoncentralT.(3.0, zeros(size(inputs, :observations))) end end @@ -147,29 +147,28 @@ end """ FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict a -multivariate normal distribution (with means matching those of the passed distributions and -identity covariance matrix) for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and +return a multivariate normal distribution (with means matching those of the passed +distributions and identity covariance matrix) for each observation. """ function FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs std_dev = ones(num_variates) - means = mean.(inputs) - return [Product(Normal.(m, std_dev)) for m in means] + means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) + return [Product(Normal.(0, std_dev)) for _ in 1:size(means, 2)] end end """ FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict 0 for each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict and return 0 for each observation. """ function FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") - samples = hcat([_handle_inputs(inputs) for _ in 1:5]...) - inputs = NamedDimsArray{(:features, :observations)}(samples) + inputs = NamedDimsArray{(:features, :observations)}(_handle_inputs(inputs)) return NamedDimsArray{(:variates, :observations)}( zeros(1, size(inputs, :observations)) ) @@ -179,13 +178,13 @@ end """ FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict a vector of 0s for each observation. The input and output will have the same dimension. +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict and return a vector of 0s for each observation. The input and output will have the +same dimension. """ function FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() do num_variates, inputs - samples = hcat([_handle_inputs(inputs) for _ in 1:5]...) - inputs = NamedDimsArray{(:features, :observations)}(samples) + inputs = NamedDimsArray{(:features, :observations)}(_handle_inputs(inputs)) return NamedDimsArray{(:variates, :observations)}( zeros(num_variates, size(inputs, :observations)) ) @@ -195,15 +194,14 @@ end """ FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict a univariate normal distribution (with zero mean and unit standard deviation) for -each observation. +A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to +predict and return a univariate normal distribution (with zero mean and unit standard +deviation) for each observation. """ function FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") - samples = hcat([_handle_inputs(inputs) for _ in 1:5]...) - inputs = NamedDimsArray{(:features, :observations)}(samples) + inputs = NamedDimsArray{(:features, :observations)}(_handle_inputs(inputs)) return NoncentralT.(3.0, zeros(size(inputs, :observations))) end end @@ -212,19 +210,19 @@ end FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict a multivariate normal distribution (with means matching those of the passed +predict adn return a multivariate normal distribution (with means matching those of the passed observations and identity covariance matrix) for each observation. """ function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() do num_variates, inputs std_dev = ones(num_variates) means = _handle_inputs(inputs) - return [Product(Normal.(m, std_dev)) for m in means] + return [Product(Normal.(0, std_dev)) for _ in 1:size(means, 2)] end end -_handle_inputs(inputs::AbstractVector{<:Sampleable}) = mean.(inputs) -_handle_inputs(inputs::AbstractMatrix) = [mean(inputs[m, :]) for m in 1:size(inputs, 2)] +_handle_inputs(inputs::AbstractVector{<:Sampleable}) = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) +_handle_inputs(inputs::AbstractMatrix) = inputs """ FakeModel @@ -317,7 +315,7 @@ end function test_interface( template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{DistributionPredictInput}; - inputs=[Normal(m, 1) for m in 1:5], outputs=rand(1, 5), + inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(1, 5), ) predictions = test_common(template, inputs, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} @@ -327,7 +325,7 @@ end function test_interface( template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; - inputs=[Normal(m, 1) for m in 1:5], outputs=rand(2, 5), + inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(2, 5), ) predictions = test_common(template, inputs, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} @@ -336,7 +334,7 @@ end function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{DistributionPredictInput}; - inputs=[Normal(m, 1) for m in 1:5], outputs=rand(1, 5), + inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(1, 5), ) predictions = test_common(template, inputs, outputs) @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} @@ -346,7 +344,7 @@ end function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; - inputs=[Normal(m, 1) for m in 1:5], outputs=rand(3, 5) + inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(3, 5) ) predictions = test_common(template, inputs, outputs) @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} @@ -356,7 +354,7 @@ end function test_interface( template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], + inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], ) predictions = test_common(template, inputs, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} @@ -372,7 +370,7 @@ end function test_interface( template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(2, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], + inputs=rand(5, 5), outputs=rand(2, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], ) predictions = test_common(template, inputs, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} @@ -386,7 +384,7 @@ end function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], + inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], ) predictions = test_common(template, inputs, outputs) @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} @@ -402,7 +400,7 @@ end function test_interface( template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(3, 5), distribution_inputs=[Normal(m, 1) for m in 1:5], + inputs=rand(5, 5), outputs=rand(3, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], ) predictions = test_common(template, inputs, outputs) @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} From 858d3683463756f4832dfffc42e8ef677593229c Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Feb 2021 14:52:59 +0000 Subject: [PATCH 15/26] Fix whitespace --- test/traits.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/traits.jl b/test/traits.jl index cda9681..4a6b009 100644 --- a/test/traits.jl +++ b/test/traits.jl @@ -5,7 +5,7 @@ struct DummyModel <: Model end estimates = (PointEstimate, DistributionEstimate) outputs = (SingleOutput, MultiOutput) - + @testset "$est, $out" for (est, out) in Iterators.product(estimates, outputs) @testset "Errors if traits are not defined" begin @@ -44,5 +44,4 @@ struct DummyModel <: Model end @test predict_input_type(DummyTemplate) == predict_input_type(DummyModel) == pinputs end end - end From 7c7d16383c58a000d4c775b08545dbda1b4d6bed Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Feb 2021 14:56:11 +0000 Subject: [PATCH 16/26] Add predict_input_type to docs --- docs/src/api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/api.md b/docs/src/api.md index 6c1b3f5..89157df 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -13,6 +13,7 @@ predict submodels estimate_type output_type +predict_input_type ``` ## Traits From 96fa7da677e48d77bc5e9ba153a62e58baf59c8a Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 23 Feb 2021 15:04:39 +0000 Subject: [PATCH 17/26] Fix docs typo --- docs/src/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/design.md b/docs/src/design.md index 4f348a2..274eb64 100644 --- a/docs/src/design.md +++ b/docs/src/design.md @@ -79,7 +79,7 @@ Here are the current [`Model`](@ref) traits in use and their possible values: The traits always agree between the [`Model`](@ref) and the [`Template`](@ref). Every [`Model`](@ref) and [`Template`](@ref) should define all the listed traits. -If left undefined, the ['PredictInputTrait'](@ref) will have the default value of [`PointPredictInput`](@ref). +If left undefined, the [`PredictInputTrait`](@ref) will have the default value of [`PointPredictInput`](@ref). This package uses traits implemented such that the trait function returns an `abstract type` (rather than an instance). That means to check a trait one uses: From 6b3c0b18a89487255c4da653ac208a500434e15b Mon Sep 17 00:00:00 2001 From: BSnelling Date: Fri, 26 Feb 2021 11:31:51 +0000 Subject: [PATCH 18/26] Apply suggestions from code review Co-authored-by: Nick Robinson --- docs/src/testutils.md | 2 +- src/Models.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/testutils.md b/docs/src/testutils.md index faf1dd9..77e6dde 100644 --- a/docs/src/testutils.md +++ b/docs/src/testutils.md @@ -12,7 +12,7 @@ test_interface ### Note on PredictInputTrait Interface Tests In the case where the [`PredictInputTrait`](@ref) is [`DistributionPredictInput`](@ref) or [`PointOrDistributionPredictInput`](@ref) the the Models API requires only that the distribution in question is `Sampleable`. -When using `Models.TestUtils.test_interface` to test a model where distributions can be passed to `predict`, the user should provide `inputs` of the distribution type appropriate to their model. +When using [`Models.TestUtils.test_interface`](@ref) to test a model where distributions can be passed to [`predict`](@ref), the user should provide `inputs` of the distribution type appropriate to their model. In the example below the `CustomModel` accepts `MvNormal` distributions to `predict`. ```julia diff --git a/src/Models.jl b/src/Models.jl index 20a34b0..92e8fad 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -55,7 +55,7 @@ return a `AbstractVector{<:Distribution}`. function predict end function predict(model::Model, inputs::AbstractVector{<:AbstractVector}) - return predict(model, hcat(inputs...)) + return predict(model, reduce(hcat, inputs)) end """ From f63f9f5c689afb3a675bdf7d2097d7e87279da2e Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Fri, 26 Feb 2021 11:47:24 +0000 Subject: [PATCH 19/26] test test utils in iterative set --- test/test_utils.jl | 66 +++++++--------------------------------------- 1 file changed, 9 insertions(+), 57 deletions(-) diff --git a/test/test_utils.jl b/test/test_utils.jl index 1a29498..4f03af4 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -1,65 +1,17 @@ @testset "test_utils.jl" begin - @testset "FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}" begin - temp = FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() + estimates = (PointEstimate, DistributionEstimate) + outputs = (SingleOutput, MultiOutput) + predict_inputs = (PointPredictInput, DistributionPredictInput, PointOrDistributionPredictInput) + + @testset "$est, $out, $pin" for (est, out, pin) in Iterators.product(estimates, outputs, predict_inputs) + temp = FakeTemplate{est, out, pin}() test_interface(temp) + end + @testset "Vector inputs case" begin temp = FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) end - - @testset "FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}" begin - temp = FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}" begin - temp = FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}" begin - temp = FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}" begin - temp = FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}" begin - temp = FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}" begin - temp = FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}" begin - temp = FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}" begin - temp = FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() - test_interface(temp) - end - - @testset "FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}" begin - temp = FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() - test_interface(temp) - end + end From 9007b2b8556271ac7ff4d04cc6b0871f28fc1038 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Fri, 26 Feb 2021 13:58:53 +0000 Subject: [PATCH 20/26] Document predict functions --- src/Models.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Models.jl b/src/Models.jl index 92e8fad..ace7bd6 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -46,6 +46,10 @@ function fit end Predict targets for the provided the collection of `inputs` and [`Model`](@ref). +A [`Model`](@ref) subtype for which the `predict_input_type(model)` is +[`PointPredictInput`](@ref) will only need to implement a `predict` function that operates +on an `AbstractMatrix` of inputs. + If the `estimate_type(model)` is [`PointEstimate`](@ref) then this function should return another `AbstractMatrix` in which each column contains the prediction for a single input. From d607f517ed46ad57536b9dbb69f59da3a1f88392 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 2 Mar 2021 09:52:24 +0000 Subject: [PATCH 21/26] Big test utils refactor --- docs/src/api.md | 1 - docs/src/design.md | 1 - docs/src/testutils.md | 2 +- src/Models.jl | 5 +- src/test_utils.jl | 316 ++++++++---------------------------------- src/traits.jl | 10 +- test/test_utils.jl | 4 +- test/traits.jl | 12 +- 8 files changed, 67 insertions(+), 284 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 89157df..5c75304 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -26,6 +26,5 @@ SingleOutput MultiOutput PredictInputTrait PointPredictInput -DistributionPredictInput PointOrDistributionPredictInput ``` diff --git a/docs/src/design.md b/docs/src/design.md index 274eb64..2f48c93 100644 --- a/docs/src/design.md +++ b/docs/src/design.md @@ -74,7 +74,6 @@ Here are the current [`Model`](@ref) traits in use and their possible values: - [`MultiOutput`](@ref): Fits and predicts on multiple outputs at a time. - [`predict_input_type`](@ref) - determines which datatypes a [`Model`](@ref) can accept at predict time. - [`PointPredictInput`](@ref): Real valued input variables accepted at predict time. - - [`DistributionPredictInput`](@ref): Distributions over input variables accepted at predict time. - [`PointOrDistributionPredictInput`](@ref): Either real valued or distributions of input variables accepted at predict time. The traits always agree between the [`Model`](@ref) and the [`Template`](@ref). diff --git a/docs/src/testutils.md b/docs/src/testutils.md index 77e6dde..6eb7c7f 100644 --- a/docs/src/testutils.md +++ b/docs/src/testutils.md @@ -11,7 +11,7 @@ test_interface ### Note on PredictInputTrait Interface Tests -In the case where the [`PredictInputTrait`](@ref) is [`DistributionPredictInput`](@ref) or [`PointOrDistributionPredictInput`](@ref) the the Models API requires only that the distribution in question is `Sampleable`. +In the case where the [`PredictInputTrait`](@ref) is [`PointOrDistributionPredictInput`](@ref) the the Models API requires only that the distribution in question is `Sampleable`. When using [`Models.TestUtils.test_interface`](@ref) to test a model where distributions can be passed to [`predict`](@ref), the user should provide `inputs` of the distribution type appropriate to their model. In the example below the `CustomModel` accepts `MvNormal` distributions to `predict`. diff --git a/src/Models.jl b/src/Models.jl index ace7bd6..eda1de8 100644 --- a/src/Models.jl +++ b/src/Models.jl @@ -6,10 +6,7 @@ export Model, Template export fit, predict, submodels, estimate_type, output_type, predict_input_type export EstimateTrait, PointEstimate, DistributionEstimate export OutputTrait, SingleOutput, MultiOutput -export PredictInputTrait, - PointPredictInput, - DistributionPredictInput, - PointOrDistributionPredictInput +export PredictInputTrait, PointPredictInput, PointOrDistributionPredictInput """ Template diff --git a/src/test_utils.jl b/src/test_utils.jl index 93f7a11..b078f4e 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -41,10 +41,10 @@ end A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict 0 for each observation. """ -function FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() - FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() do num_variates, inputs +function FakeTemplate{PointEstimate, SingleOutput, I}() where {I<:PredictInputTrait} + FakeTemplate{PointEstimate, SingleOutput, I}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") - inputs = NamedDimsArray{(:features, :observations)}(inputs) + inputs = _handle_inputs(inputs) return NamedDimsArray{(:variates, :observations)}( zeros(1, size(inputs, :observations)) ) @@ -57,9 +57,9 @@ end A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a vector of 0s for each observation. The input and output will have the same dimension. """ -function FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() - FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() do num_variates, inputs - inputs = NamedDimsArray{(:features, :observations)}(inputs) +function FakeTemplate{PointEstimate, MultiOutput, I}() where {I<:PredictInputTrait} + FakeTemplate{PointEstimate, MultiOutput, I}() do num_variates, inputs + inputs = _handle_inputs(inputs) return NamedDimsArray{(:variates, :observations)}( zeros(num_variates, size(inputs, :observations)) ) @@ -73,10 +73,10 @@ A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to p univariate normal distribution (with zero mean and unit standard deviation) for each observation. """ -function FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() - FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, SingleOutput, I}() where {I<:PredictInputTrait} + FakeTemplate{DistributionEstimate, SingleOutput, I}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") - inputs = NamedDimsArray{(:features, :observations)}(inputs) + inputs = _handle_inputs(inputs) return NoncentralT.(3.0, zeros(size(inputs, :observations))) end end @@ -88,141 +88,26 @@ A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to p multivariate normal distribution (with zero-vector mean and identity covariance matrix) for each observation. """ -function FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() - FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, MultiOutput, I}() where {I<:PredictInputTrait} + FakeTemplate{DistributionEstimate, MultiOutput, I}() do num_variates, inputs std_dev = ones(num_variates) + inputs = _handle_inputs(inputs) return [Product(Normal.(0, std_dev)) for _ in 1:size(inputs, 2)] end end """ - FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() + _handle_inputs(inputs::AbstractMatrix) + _handle_inputs(inputs::AbstractVector{<:Sampleable}) -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and -return 0 for each observation. +Process the inputs to `predict` appropriately depending on whether they are real valued or +distributions over input variables. """ -function FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() - FakeTemplate{PointEstimate, SingleOutput, DistributionPredictInput}() do num_variates, inputs - @assert(num_variates == 1, "$num_variates != 1") - means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) - inputs = NamedDimsArray{(:features, :observations)}(means) - return NamedDimsArray{(:variates, :observations)}( - zeros(1, size(inputs, :observations)) - ) - end -end - -""" - FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and -return a vector of 0s for each observation. The input and output will have the same dimension. -""" -function FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() - FakeTemplate{PointEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs - means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) - inputs = NamedDimsArray{(:features, :observations)}(means) - return NamedDimsArray{(:variates, :observations)}( - zeros(num_variates, size(inputs, :observations)) - ) - end +function _handle_inputs(inputs::AbstractVector{<:Sampleable}) + return NamedDimsArray{(:features, :observations)}(hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...)) end -""" - FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and -return a univariate normal distribution (with zero mean and unit standard deviation) for each -observation. -""" -function FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() - FakeTemplate{DistributionEstimate, SingleOutput, DistributionPredictInput}() do num_variates, inputs - @assert(num_variates == 1, "$num_variates != 1") - means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) - inputs = NamedDimsArray{(:features, :observations)}(means) - return NoncentralT.(3.0, zeros(size(inputs, :observations))) - end -end - -""" - FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept distribution inputs to predict and -return a multivariate normal distribution (with means matching those of the passed -distributions and identity covariance matrix) for each observation. -""" -function FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() - FakeTemplate{DistributionEstimate, MultiOutput, DistributionPredictInput}() do num_variates, inputs - std_dev = ones(num_variates) - means = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) - return [Product(Normal.(0, std_dev)) for _ in 1:size(means, 2)] - end -end - -""" - FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict and return 0 for each observation. -""" -function FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() - FakeTemplate{PointEstimate, SingleOutput, PointOrDistributionPredictInput}() do num_variates, inputs - @assert(num_variates == 1, "$num_variates != 1") - inputs = NamedDimsArray{(:features, :observations)}(_handle_inputs(inputs)) - return NamedDimsArray{(:variates, :observations)}( - zeros(1, size(inputs, :observations)) - ) - end -end - -""" - FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict and return a vector of 0s for each observation. The input and output will have the -same dimension. -""" -function FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() - FakeTemplate{PointEstimate, MultiOutput, PointOrDistributionPredictInput}() do num_variates, inputs - inputs = NamedDimsArray{(:features, :observations)}(_handle_inputs(inputs)) - return NamedDimsArray{(:variates, :observations)}( - zeros(num_variates, size(inputs, :observations)) - ) - end -end - -""" - FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict and return a univariate normal distribution (with zero mean and unit standard -deviation) for each observation. -""" -function FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() - FakeTemplate{DistributionEstimate, SingleOutput, PointOrDistributionPredictInput}() do num_variates, inputs - @assert(num_variates == 1, "$num_variates != 1") - inputs = NamedDimsArray{(:features, :observations)}(_handle_inputs(inputs)) - return NoncentralT.(3.0, zeros(size(inputs, :observations))) - end -end - -""" - FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() - -A [`Template`](@ref) whose [`Model`](@ref) will accept real-valued or distribution inputs to -predict adn return a multivariate normal distribution (with means matching those of the passed -observations and identity covariance matrix) for each observation. -""" -function FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() - FakeTemplate{DistributionEstimate, MultiOutput, PointOrDistributionPredictInput}() do num_variates, inputs - std_dev = ones(num_variates) - means = _handle_inputs(inputs) - return [Product(Normal.(0, std_dev)) for _ in 1:size(means, 2)] - end -end - -_handle_inputs(inputs::AbstractVector{<:Sampleable}) = hcat([mean(inputs[i]) for i in 1:size(inputs, 1)]...) -_handle_inputs(inputs::AbstractMatrix) = inputs +_handle_inputs(inputs::AbstractMatrix) = NamedDimsArray{(:features, :observations)}(inputs) """ FakeModel @@ -262,156 +147,67 @@ StatsBase.predict(m::FakeModel, inputs::AbstractVector{<:Sampleable}) = m.predic Test that subtypes of [`Template`](@ref) and [`Model`](@ref) implement the expected API. Can be used as an initial test to verify the API has been correctly implemented. """ -function test_interface(template::Template; kwargs...) - @testset "Models API Interface Test: $(nameof(typeof(template)))" begin - test_interface( - template, - estimate_type(template), - output_type(template), - predict_input_type(template); - kwargs... - ) +function _default_outputs(template) + if output_type(template) == SingleOutput + return rand(1, 5) + elseif output_type(template) == MultiOutput + return rand(2, 5) + else + println("Invalid output trait") end end function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointPredictInput}; - inputs=rand(5, 5), outputs=rand(1, 5), -) - predictions = test_common(template, inputs, outputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) - @test size(predictions, 1) == 1 -end - -function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointPredictInput}; - inputs=rand(5, 5), outputs=rand(2, 5), -) - predictions = test_common(template, inputs, outputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) -end - -function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointPredictInput}; - inputs=rand(5, 5), outputs=rand(1, 5), + template::Template; + inputs=rand(5,5), + outputs=_default_outputs(template), + distribution_inputs=[MvNormal(5, m) for m in 1:5], + kwargs... ) - predictions = test_common(template, inputs, outputs) - @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} - @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) -end - -function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointPredictInput}; - inputs=rand(5, 5), outputs=rand(3, 5) -) - predictions = test_common(template, inputs, outputs) - @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} - @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) -end - -function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{DistributionPredictInput}; - inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(1, 5), -) - predictions = test_common(template, inputs, outputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) - @test size(predictions, 1) == 1 + @testset "Models API Interface Test: $(nameof(typeof(template)))" begin + predictions = test_common(template, inputs, outputs) + test_estimate_type(estimate_type(template), predictions, outputs) + test_output_type(output_type(template), predictions, outputs) + test_predict_input_type(predict_input_type(template), template, predictions, outputs, inputs, distribution_inputs) + end end -function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; - inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(2, 5), -) - predictions = test_common(template, inputs, outputs) +function test_estimate_type(::Type{PointEstimate}, predictions, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} @test size(predictions) == size(outputs) end -function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{DistributionPredictInput}; - inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(1, 5), -) - predictions = test_common(template, inputs, outputs) - @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} +function test_estimate_type(::Type{DistributionEstimate}, predictions, outputs) + @test predictions isa AbstractVector{<:ContinuousDistribution} @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) end -function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{DistributionPredictInput}; - inputs=[MvNormal(5, m) for m in 1:5], outputs=rand(3, 5) -) - predictions = test_common(template, inputs, outputs) - @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} - @test length(predictions) == size(outputs, 2) +function test_output_type(::Type{SingleOutput}, predictions, outputs) @test all(length.(predictions) .== size(outputs, 1)) + @test all(length.(predictions) .== 1) end -function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{SingleOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], -) - predictions = test_common(template, inputs, outputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) - @test size(predictions, 1) == 1 - - model = fit(template, outputs, inputs) - predictions = predict(model, distribution_inputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) - @test size(predictions, 1) == 1 -end - -function test_interface( - template::Template, ::Type{PointEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(2, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], -) - predictions = test_common(template, inputs, outputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) - - model = fit(template, outputs, inputs) - predictions = predict(model, distribution_inputs) - @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} - @test size(predictions) == size(outputs) +function test_output_type(::Type{MultiOutput}, predictions, outputs) + if eltype(predictions) <: Distribution + @test all(length.(predictions) .== size(outputs, 1)) + @test all(length.(predictions) .> 1) + else + @test size(predictions, 1) == size(outputs, 1) + @test size(predictions, 1) > 1 + end end -function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{SingleOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(1, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], -) - predictions = test_common(template, inputs, outputs) - @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} - @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) - +function test_predict_input_type(::Type{PointPredictInput}, template, predictions, outputs, inputs, distribution_inputs) model = fit(template, outputs, inputs) - predictions = predict(model, distribution_inputs) - @test predictions isa AbstractVector{<:ContinuousUnivariateDistribution} - @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) + @test hasmethod(Models.predict, (typeof(model), typeof(inputs))) end -function test_interface( - template::Template, ::Type{DistributionEstimate}, ::Type{MultiOutput}, ::Type{PointOrDistributionPredictInput}; - inputs=rand(5, 5), outputs=rand(3, 5), distribution_inputs=[MvNormal(5, m) for m in 1:5], -) - predictions = test_common(template, inputs, outputs) - @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} - @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) - +function test_predict_input_type(::Type{PointOrDistributionPredictInput}, template, predictions, outputs, inputs, distribution_inputs) model = fit(template, outputs, inputs) + @test hasmethod(Models.predict, (typeof(model), typeof(distribution_inputs))) predictions = predict(model, distribution_inputs) - @test predictions isa AbstractVector{<:ContinuousMultivariateDistribution} - @test length(predictions) == size(outputs, 2) - @test all(length.(predictions) .== size(outputs, 1)) + test_estimate_type(estimate_type(template), predictions, outputs) + test_output_type(output_type(template), predictions, outputs) end function test_names(template, model) diff --git a/src/traits.jl b/src/traits.jl index 4554d5e..ad1c044 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -63,8 +63,7 @@ output_type(T::Type) = throw(MethodError(output_type, (T,))) # to prevent recur PredictInputTrait The `PredictInputTrait` specifies if the model supports point or distribution injections to predict, -denoted by [`PointPredictInput`](@ref) or [`DistributionPredictInput`](@ref), respectively. A model can -also be implemented in such a way as to allow [`PointOrDistributionPredictInput`](@ref). +denoted by [`PointPredictInput`](@ref) or [`PointOrDistributionPredictInput`](@ref). """ abstract type PredictInputTrait end @@ -75,13 +74,6 @@ Specifies that the [`Model`](@ref) accepts real-valued input variables to `predi """ abstract type PointPredictInput <: PredictInputTrait end -""" - DistributionPredictInput <: PredictInputTrait - -Specifies that the [`Model`](@ref) accepts a distribution over the input variables to `predict`. -""" -abstract type DistributionPredictInput <: PredictInputTrait end - """ PointOrDistributionPredictInput <: PredictInputTrait diff --git a/test/test_utils.jl b/test/test_utils.jl index 4f03af4..973bd92 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -2,7 +2,7 @@ estimates = (PointEstimate, DistributionEstimate) outputs = (SingleOutput, MultiOutput) - predict_inputs = (PointPredictInput, DistributionPredictInput, PointOrDistributionPredictInput) + predict_inputs = (PointPredictInput, PointOrDistributionPredictInput) @testset "$est, $out, $pin" for (est, out, pin) in Iterators.product(estimates, outputs, predict_inputs) temp = FakeTemplate{est, out, pin}() @@ -13,5 +13,5 @@ temp = FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) end - + end diff --git a/test/traits.jl b/test/traits.jl index 4a6b009..9e6016f 100644 --- a/test/traits.jl +++ b/test/traits.jl @@ -35,13 +35,13 @@ struct DummyModel <: Model end @test predict_input_type(DummyTemplate) == predict_input_type(DummyModel) == PointPredictInput end - predict_input = (DistributionPredictInput, PointOrDistributionPredictInput) - - @testset "$pinputs is defined" for pinputs in predict_input - predict_input_type(m::Type{<:DummyTemplate}) = pinputs - predict_input_type(m::Type{<:DummyModel}) = pinputs + @testset "PointOrDistributionPredictInput is defined" begin + predict_input_type(m::Type{<:DummyTemplate}) = PointOrDistributionPredictInput + predict_input_type(m::Type{<:DummyModel}) = PointOrDistributionPredictInput - @test predict_input_type(DummyTemplate) == predict_input_type(DummyModel) == pinputs + @test predict_input_type(DummyTemplate) == + predict_input_type(DummyModel) == + PointOrDistributionPredictInput end end end From fbb3242a6ee908b60c73b01d185bee5e71b87fb4 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 2 Mar 2021 14:04:56 +0000 Subject: [PATCH 22/26] With PredictInputTrait as type parameter --- src/test_utils.jl | 25 ++++++++++++------------- test/traits.jl | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index b078f4e..762fac0 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -147,16 +147,6 @@ StatsBase.predict(m::FakeModel, inputs::AbstractVector{<:Sampleable}) = m.predic Test that subtypes of [`Template`](@ref) and [`Model`](@ref) implement the expected API. Can be used as an initial test to verify the API has been correctly implemented. """ -function _default_outputs(template) - if output_type(template) == SingleOutput - return rand(1, 5) - elseif output_type(template) == MultiOutput - return rand(2, 5) - else - println("Invalid output trait") - end -end - function test_interface( template::Template; inputs=rand(5,5), @@ -168,7 +158,16 @@ function test_interface( predictions = test_common(template, inputs, outputs) test_estimate_type(estimate_type(template), predictions, outputs) test_output_type(output_type(template), predictions, outputs) - test_predict_input_type(predict_input_type(template), template, predictions, outputs, inputs, distribution_inputs) + test_predict_input_type(predict_input_type(template), template, outputs, inputs, distribution_inputs) + end +end + +function _default_outputs(template) + @assert(output_type(template) <: OutputTrait, "Invalid OutputTrait") + if output_type(template) == SingleOutput + return rand(1, 5) + else output_type(template) == MultiOutput + return rand(2, 5) end end @@ -197,12 +196,12 @@ function test_output_type(::Type{MultiOutput}, predictions, outputs) end end -function test_predict_input_type(::Type{PointPredictInput}, template, predictions, outputs, inputs, distribution_inputs) +function test_predict_input_type(::Type{PointPredictInput}, template, outputs, inputs, distribution_inputs) model = fit(template, outputs, inputs) @test hasmethod(Models.predict, (typeof(model), typeof(inputs))) end -function test_predict_input_type(::Type{PointOrDistributionPredictInput}, template, predictions, outputs, inputs, distribution_inputs) +function test_predict_input_type(::Type{PointOrDistributionPredictInput}, template, outputs, inputs, distribution_inputs) model = fit(template, outputs, inputs) @test hasmethod(Models.predict, (typeof(model), typeof(distribution_inputs))) predictions = predict(model, distribution_inputs) diff --git a/test/traits.jl b/test/traits.jl index 9e6016f..65ac73d 100644 --- a/test/traits.jl +++ b/test/traits.jl @@ -29,7 +29,7 @@ struct DummyModel <: Model end end - @testset "PredictInput Trait" begin + @testset "PredictInputTrait" begin @testset "Default" begin @test predict_input_type(DummyTemplate) == predict_input_type(DummyModel) == PointPredictInput From 4462c1976035312605ae6c6d64b7dffcf470fe41 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 2 Mar 2021 14:42:08 +0000 Subject: [PATCH 23/26] Remove PredictInputTrait as type parameter --- src/test_utils.jl | 46 ++++++++++++++++++++++------------------------ test/test_utils.jl | 25 +++++++++++++++++++++---- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 762fac0..4cb8846 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -16,7 +16,7 @@ export FakeModel, FakeTemplate export test_interface """ - FakeTemplate{E <: EstimateTrait, O <: OutputTrait, I <: PredictInputTrait} <: Template + FakeTemplate{E <: EstimateTrait, O <: OutputTrait} <: Template This template is a [test double](https://en.wikipedia.org/wiki/Test_double) for testing purposes. It should be defined (before fitting) with a `predictor`, which can be changed by @@ -31,18 +31,18 @@ mutating the field. - [`fit`](@ref) does not learn anything, it just creates an instance of the corresponding [`Model`](@ref). - [`predict`](@ref) applies the `predictor` to the inputs. """ -mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait, I<:PredictInputTrait} <: Template +mutable struct FakeTemplate{E<:EstimateTrait, O<:OutputTrait} <: Template predictor::Function end """ - FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() + FakeTemplate{PointEstimate, SingleOutput}() A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict 0 for each observation. """ -function FakeTemplate{PointEstimate, SingleOutput, I}() where {I<:PredictInputTrait} - FakeTemplate{PointEstimate, SingleOutput, I}() do num_variates, inputs +function FakeTemplate{PointEstimate, SingleOutput}() + FakeTemplate{PointEstimate, SingleOutput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") inputs = _handle_inputs(inputs) return NamedDimsArray{(:variates, :observations)}( @@ -52,13 +52,13 @@ function FakeTemplate{PointEstimate, SingleOutput, I}() where {I<:PredictInputTr end """ - FakeTemplate{PointEstimate, MultiOutput, PointPredictInput}() + FakeTemplate{PointEstimate, MultiOutput}() A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a vector of 0s for each observation. The input and output will have the same dimension. """ -function FakeTemplate{PointEstimate, MultiOutput, I}() where {I<:PredictInputTrait} - FakeTemplate{PointEstimate, MultiOutput, I}() do num_variates, inputs +function FakeTemplate{PointEstimate, MultiOutput}() + FakeTemplate{PointEstimate, MultiOutput}() do num_variates, inputs inputs = _handle_inputs(inputs) return NamedDimsArray{(:variates, :observations)}( zeros(num_variates, size(inputs, :observations)) @@ -67,14 +67,14 @@ function FakeTemplate{PointEstimate, MultiOutput, I}() where {I<:PredictInputTra end """ - FakeTemplate{DistributionEstimate, SingleOutput, PointPredictInput}() + FakeTemplate{DistributionEstimate, SingleOutput}() A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a univariate normal distribution (with zero mean and unit standard deviation) for each observation. """ -function FakeTemplate{DistributionEstimate, SingleOutput, I}() where {I<:PredictInputTrait} - FakeTemplate{DistributionEstimate, SingleOutput, I}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, SingleOutput}() + FakeTemplate{DistributionEstimate, SingleOutput}() do num_variates, inputs @assert(num_variates == 1, "$num_variates != 1") inputs = _handle_inputs(inputs) return NoncentralT.(3.0, zeros(size(inputs, :observations))) @@ -82,14 +82,14 @@ function FakeTemplate{DistributionEstimate, SingleOutput, I}() where {I<:Predict end """ - FakeTemplate{DistributionEstimate, MultiOutput, PointPredictInput}() + FakeTemplate{DistributionEstimate, MultiOutput}() A [`Template`](@ref) whose [`Model`](@ref) will accept real value variables to predict a multivariate normal distribution (with zero-vector mean and identity covariance matrix) for each observation. """ -function FakeTemplate{DistributionEstimate, MultiOutput, I}() where {I<:PredictInputTrait} - FakeTemplate{DistributionEstimate, MultiOutput, I}() do num_variates, inputs +function FakeTemplate{DistributionEstimate, MultiOutput}() + FakeTemplate{DistributionEstimate, MultiOutput}() do num_variates, inputs std_dev = ones(num_variates) inputs = _handle_inputs(inputs) return [Product(Normal.(0, std_dev)) for _ in 1:size(inputs, 2)] @@ -114,28 +114,26 @@ _handle_inputs(inputs::AbstractMatrix) = NamedDimsArray{(:features, :observation A fake Model for testing purposes. See [`FakeTemplate`](@ref) for details. """ -mutable struct FakeModel{E<:EstimateTrait, O<:OutputTrait, I<:PredictInputTrait} <: Model +mutable struct FakeModel{E<:EstimateTrait, O<:OutputTrait} <: Model predictor::Function num_variates::Int end -Models.estimate_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = E -Models.output_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = O -Models.predict_input_type(::Type{<:FakeModel{E, O, I}}) where {E, O, I} = I +Models.estimate_type(::Type{<:FakeModel{E, O}}) where {E, O} = E +Models.output_type(::Type{<:FakeModel{E, O}}) where {E, O} = O -Models.estimate_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = E -Models.output_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = O -Models.predict_input_type(::Type{<:FakeTemplate{E, O, I}}) where {E, O, I} = I +Models.estimate_type(::Type{<:FakeTemplate{E, O}}) where {E, O} = E +Models.output_type(::Type{<:FakeTemplate{E, O}}) where {E, O} = O function StatsBase.fit( - template::FakeTemplate{E, O, I}, + template::FakeTemplate{E, O}, outputs, inputs, weights=uweights(Float32, size(outputs, 2)) -) where {E, O, I} +) where {E, O} outputs = NamedDimsArray{(:variates, :observations)}(outputs) num_variates = size(outputs, :variates) - return FakeModel{E, O, I}(template.predictor, num_variates) + return FakeModel{E, O}(template.predictor, num_variates) end StatsBase.predict(m::FakeModel, inputs::AbstractMatrix) = m.predictor(m.num_variates, inputs) diff --git a/test/test_utils.jl b/test/test_utils.jl index 973bd92..079dbb8 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -2,16 +2,33 @@ estimates = (PointEstimate, DistributionEstimate) outputs = (SingleOutput, MultiOutput) - predict_inputs = (PointPredictInput, PointOrDistributionPredictInput) - @testset "$est, $out, $pin" for (est, out, pin) in Iterators.product(estimates, outputs, predict_inputs) - temp = FakeTemplate{est, out, pin}() + @testset "$est, $out, PointPredictInput" for (est, out) in Iterators.product(estimates, outputs) + temp = FakeTemplate{est, out}() test_interface(temp) end @testset "Vector inputs case" begin - temp = FakeTemplate{PointEstimate, SingleOutput, PointPredictInput}() + temp = FakeTemplate{PointEstimate, SingleOutput}() test_interface(temp; inputs=[rand(5), rand(5)], outputs=rand(1, 2)) end + @testset "$est, $out, PointOrDistributionPredictInput" for (est, out) in Iterators.product(estimates, outputs) + Models.predict_input_type(m::Type{<:FakeTemplate}) = PointOrDistributionPredictInput + Models.predict_input_type(m::Type{<:FakeModel}) = PointOrDistributionPredictInput + + temp = FakeTemplate{est, out}() + test_interface(temp) + end + + @testset "Vector inputs case" begin + temp = FakeTemplate{PointEstimate, SingleOutput}() + test_interface( + temp; + inputs=[rand(5), rand(5)], + outputs=rand(1, 2), + distribution_inputs=[MvNormal(5, m) for m in 1:2] + ) + end + end From 40bdc6a0277600271f559ea084ad7f90dee89df5 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 2 Mar 2021 15:16:44 +0000 Subject: [PATCH 24/26] Refactor _default_outputs to dispatch --- src/test_utils.jl | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 4cb8846..8e833c9 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -160,14 +160,9 @@ function test_interface( end end -function _default_outputs(template) - @assert(output_type(template) <: OutputTrait, "Invalid OutputTrait") - if output_type(template) == SingleOutput - return rand(1, 5) - else output_type(template) == MultiOutput - return rand(2, 5) - end -end +_default_outputs(template) = _default_outputs(output_type(template), template) +_default_outputs(::Type{SingleOutput}, template) = rand(1, 5) +_default_outputs(::Type{MultiOutput}, template) = rand(2, 5) function test_estimate_type(::Type{PointEstimate}, predictions, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} From 2af04dab2e429b73e6ec1e9ca068c030623a8061 Mon Sep 17 00:00:00 2001 From: Branwen Snelling Date: Tue, 2 Mar 2021 15:35:39 +0000 Subject: [PATCH 25/26] Another refactor to _default_outputs --- src/test_utils.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 8e833c9..5722326 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -160,9 +160,9 @@ function test_interface( end end -_default_outputs(template) = _default_outputs(output_type(template), template) -_default_outputs(::Type{SingleOutput}, template) = rand(1, 5) -_default_outputs(::Type{MultiOutput}, template) = rand(2, 5) +_default_outputs(template) = _default_outputs(output_type(template)) +_default_outputs(::Type{SingleOutput}) = rand(1, 5) +_default_outputs(::Type{MultiOutput}) = rand(2, 5) function test_estimate_type(::Type{PointEstimate}, predictions, outputs) @test predictions isa NamedDimsArray{(:variates, :observations), <:Real, 2} From 92523214fa55f65514681414f952078ff02751ad Mon Sep 17 00:00:00 2001 From: BSnelling Date: Tue, 2 Mar 2021 17:22:48 +0000 Subject: [PATCH 26/26] Apply suggestions from code review Co-authored-by: Sean Lovett --- src/traits.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/traits.jl b/src/traits.jl index ad1c044..a84350c 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -62,7 +62,7 @@ output_type(T::Type) = throw(MethodError(output_type, (T,))) # to prevent recur """ PredictInputTrait -The `PredictInputTrait` specifies if the model supports point or distribution injections to predict, +The `PredictInputTrait` specifies if the model supports point or distribution inputs to `predict`, denoted by [`PointPredictInput`](@ref) or [`PointOrDistributionPredictInput`](@ref). """ abstract type PredictInputTrait end @@ -77,7 +77,7 @@ abstract type PointPredictInput <: PredictInputTrait end """ PointOrDistributionPredictInput <: PredictInputTrait -Specifies that the [`Model`](@ref) accepts real-values or a distribution over the input +Specifies that the [`Model`](@ref) accepts real-values or a joint distribution over the input variables to `predict`. """ abstract type PointOrDistributionPredictInput <: PredictInputTrait end