add missingness management for standardizer

src/transformers/other_transformers/standardizer.jl

@@ -1,3 +1,13 @@
+# Utilities
+
+get_mean_and_std(v::AbstractVector{T}) where T <: Real = mean(v), std(v)
+
+function get_mean_and_std(v::AbstractVector{T}) where T <: Union{Missing, Real}
+    v_ = skipmissing(v)
+    return mean(v_), std(v_)
+end
+
+# Univariate Standardizer
 
 """
     UnivariateStandardizer()
@@ -11,11 +21,11 @@ This model may be deprecated in the future. Consider using
 mutable struct UnivariateStandardizer <: Unsupervised end
 
 function MMI.fit(transformer::UnivariateStandardizer, verbosity::Int,
-             v::AbstractVector{T}) where T<:Real
-    stdv = std(v)
+             v::AbstractVector{T}) where T<:Union{Real, Missing}
+    meanv, stdv = get_mean_and_std(v)
     stdv > eps(typeof(stdv)) ||
         @warn "Extremely small standard deviation encountered in standardization."
-    fitresult = (mean(v), stdv)
+    fitresult = (meanv, stdv)
     cache = nothing
     report = NamedTuple()
     return fitresult, cache, report
@@ -31,6 +41,8 @@ function MMI.transform(transformer::UnivariateStandardizer, fitresult, x::Real)
     return (x - mu)/sigma
 end
 
+MMI.transform(transformer::UnivariateStandardizer, fitresult, x::Missing) = missing
+
 # for transforming vector:
 MMI.transform(transformer::UnivariateStandardizer, fitresult, v) =
               [transform(transformer, fitresult, x) for x in v]
@@ -41,6 +53,8 @@ function MMI.inverse_transform(transformer::UnivariateStandardizer, fitresult, y
     return mu + y*sigma
 end
 
+MMI.inverse_transform(transformer::UnivariateStandardizer, fitresult, y::Missing) = missing
+
 # for vectors:
 MMI.inverse_transform(transformer::UnivariateStandardizer, fitresult, w) =
     [inverse_transform(transformer, fitresult, y) for y in w]
@@ -83,7 +97,6 @@ function MMI.clean!(transformer::Standardizer)
 end
 
 function MMI.fit(transformer::Standardizer, verbosity::Int, X)
-
     # if not a table, it must be an abstract vector, eltpye AbstractFloat:
     is_univariate = !Tables.istable(X)
 
@@ -106,9 +119,9 @@ function MMI.fit(transformer::Standardizer, verbosity::Int, X)
     all_features = Tables.schema(X).names
     feature_scitypes =
         collect(elscitype(selectcols(X, c)) for c in all_features)
-    scitypes = Vector{Type}([Continuous])
-    transformer.ordered_factor && push!(scitypes, OrderedFactor)
-    transformer.count && push!(scitypes, Count)
+    scitypes = Vector{Type}([Union{Missing, Continuous}])
+    transformer.ordered_factor && push!(scitypes, Union{Missing, OrderedFactor})
+    transformer.count && push!(scitypes, Union{Missing, Count})
     AllowedScitype = Union{scitypes...}
 
     # determine indices of all_features to be transformed
@@ -148,7 +161,7 @@ function MMI.fit(transformer::Standardizer, verbosity::Int, X)
     # fit each feature and add result to above dict
     verbosity > 1 && @info "Features standarized: "
     for j in cols_to_fit
-        col_data = if (feature_scitypes[j] <: OrderedFactor)
+        col_data = if (feature_scitypes[j] <: Union{Missing, OrderedFactor})
             coerce(selectcols(X, j), Continuous)
         else
             selectcols(X, j)
@@ -259,7 +272,8 @@ Use this model to standardize (whiten) a `Continuous` vector, or
 relevant columns of a table. The rescalings applied by this
 transformer to new data are always those learned during the training
 phase, which are generally different from what would actually
-standardize the new data.
+standardize the new data. Missing values are ignored by the transformer and
+returned as missing.
 
 
 # Training data

test/transformers/other_transformers/standardizer.jl

@@ -111,6 +111,28 @@
     @test Xnew[4] == X[4]
     @test Xnew[5] == X[5]
 
+    # Test missingness in Standardizer
+    X = (
+        OverallQual  = [1, 3, missing, 2, missing, 1],
+        GrLivArea    = [2., 1., missing, missing, missing],
+        Neighborhood = categorical(['1', missing, '2', '3', '3'], ordered=true)
+    )
+    stand = Standardizer(ordered_factor=true, count=true)
+    f,   = MLJBase.fit(stand, 0, X)
+    Xt   = MLJBase.transform(stand, f, X)
+    for col in values(Xt)
+        meanv, stdv = MLJTransforms.get_mean_and_std(col)
+        @test meanv ≈ 0 atol=1e-10
+        @test stdv ≈ 1 atol=1e-10
+    end
+    # Test inverse transform when missingness is present
+    stand = Standardizer()
+    f,   = MLJBase.fit(stand, 0, X)
+    Xt   = MLJBase.transform(stand, f, X)
+    Xi   = MLJBase.inverse_transform(stand, f, Xt)
+    @test isequal(Xi.GrLivArea, X.GrLivArea)
+
+
     # infos = MLJTransforms.info_dict(stand)
 
     # @test infos[:name] == "Standardizer"
@@ -126,4 +148,10 @@
     fp = MLJBase.fitted_params(stand, f)
     @test fp.mean ≈ 2.0
     @test fp.std ≈ 2.0
+    # univariate case with missingness
+    f, _, _   = MLJBase.fit(stand, 0, [0, 2, 4, missing])
+    xt = MLJBase.transform(stand, f, [0,4,8, missing])
+    @test isequal(xt, [-1.0,1.0,3.0, missing])
+    xi = MLJBase.inverse_transform(stand, f, [-1.0,1.0,3.0, missing])
+    @test isequal(xi, [0., 4., 8., missing])
 end