Convert inverse transform to use warnings for cases where bijection is broken and test that we get nulls when bijection is broken

Author: JohnnyC08

category_encoders/basen.py

@@ -7,6 +7,7 @@
 from sklearn.base import BaseEstimator, TransformerMixin
 from category_encoders.ordinal import OrdinalEncoder
 import category_encoders.utils as util
+import warnings
 
 __author__ = 'willmcginnis'
 
@@ -276,23 +277,17 @@ def inverse_transform(self, X_in):
         if not self.cols:
             return X if self.return_df else X.values
 
-        if self.handle_unknown == 'value':
-            for col in self.cols:
-                if any(X[col] == -1):
-                    raise ValueError("inverse_transform is not supported because transform impute "
-                                     "the unknown category -1 when encode %s" % (col,))
-
-        if self.handle_unknown == 'return_nan':
-            for col in self.cols:
-                if X[col].isnull().any():
-                    raise ValueError("inverse_transform is not supported because transform impute "
-                                     "the unknown category nan when encode %s" % (col,))
-
         for switch in self.ordinal_encoder.mapping:
             column_mapping = switch.get('mapping')
             inverse = pd.Series(data=column_mapping.index, index=column_mapping.get_values())
             X[switch.get('col')] = X[switch.get('col')].map(inverse).astype(switch.get('data_type'))
 
+            if self.handle_unknown == 'return_nan' and self.handle_missing == 'return_nan':
+                for col in self.cols:
+                    if X[switch.get('col')].isnull().any():
+                        warnings.warn("inverse_transform is not supported because transform impute "
+                                      "the unknown category nan when encode %s" % (col,))
+
         return X if self.return_df else X.values
 
     def calc_required_digits(self, values):
@@ -356,10 +351,7 @@ def basen_to_integer(self, X, cols, base):
 
         for col in cols:
             col_list = [col0 for col0 in out_cols if str(col0).startswith(str(col))]
-            for col0 in col_list:
-                if any(X[col0].isnull()):
-                    raise ValueError("inverse_transform is not supported because transform impute"
-                                     "the unknown category -1 when encode %s" % (col,))
+
             if base == 1:
                 value_array = np.array([int(col0.split('_')[-1]) for col0 in col_list])
             else:

category_encoders/one_hot.py

@@ -1,7 +1,7 @@
 """One-hot or dummy coding"""
 import numpy as np
 import pandas as pd
-import copy
+import warnings
 from sklearn.base import BaseEstimator, TransformerMixin
 from category_encoders.ordinal import OrdinalEncoder
 import category_encoders.utils as util
@@ -306,23 +306,17 @@ def inverse_transform(self, X_in):
         if not self.cols:
             return X if self.return_df else X.values
 
-        if self.handle_unknown == 'value':
-            for col in self.cols:
-                if any(X[col] == -1):
-                    raise ValueError("inverse_transform is not supported because transform impute "
-                                     "the unknown category -1 when encode %s"%(col,))
-
-        if self.handle_unknown == 'return_nan':
-            for col in self.cols:
-                if X[col].isnull().any():
-                    raise ValueError("inverse_transform is not supported because transform impute "
-                                     "the unknown category nan when encode %s" % (col,))
-
         for switch in self.ordinal_encoder.mapping:
             column_mapping = switch.get('mapping')
             inverse = pd.Series(data=column_mapping.index, index=column_mapping.get_values())
             X[switch.get('col')] = X[switch.get('col')].map(inverse).astype(switch.get('data_type'))
 
+            if self.handle_unknown == 'return_nan' and self.handle_missing == 'return_nan':
+                for col in self.cols:
+                    if X[switch.get('col')].isnull().any():
+                        warnings.warn("inverse_transform is not supported because transform impute "
+                                      "the unknown category nan when encode %s" % (col,))
+
         return X if self.return_df else X.values
 
     def get_dummies(self, X_in):

category_encoders/tests/test_basen.py

@@ -2,6 +2,7 @@
 from unittest2 import TestCase  # or `from unittest import ...` if on Python 3.4+
 import numpy as np
 import category_encoders as encoders
+import warnings
 
 
 class TestBaseNEncoder(TestCase):
@@ -84,3 +85,59 @@ def test_HandleUnknown_HaveOnlyKnown_ExpectSecondColumn(self):
         self.assertEqual(2, result.shape[0])
         self.assertListEqual([0, 0, 1], result.iloc[0, :].tolist())
         self.assertListEqual([0, 1, 0], result.iloc[1, :].tolist())
+
+    def test_inverse_transform_HaveNanInTrainAndHandleMissingValue_ExpectReturnedWithNan(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+
+        enc = encoders.BaseNEncoder(handle_missing='value', handle_unknown='value')
+        result = enc.fit_transform(train)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(train, original)
+
+    def test_inverse_transform_HaveNanInTrainAndHandleMissingReturnNan_ExpectReturnedWithNan(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+
+        enc = encoders.BaseNEncoder(handle_missing='return_nan', handle_unknown='value')
+        result = enc.fit_transform(train)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(train, original)
+
+    def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+        test = pd.DataFrame({'city': ['chicago', 'los angeles']})
+
+        enc = encoders.BaseNEncoder(handle_missing='return_nan', handle_unknown='return_nan')
+        enc.fit(train)
+        result = enc.transform(test)
+
+        with warnings.catch_warnings(record=True) as w:
+            enc.inverse_transform(result)
+
+            self.assertEqual(2, len(w))
+            self.assertEqual('inverse_transform is not supported because transform impute '
+                             'the unknown category nan when encode city', str(w[1].message))
+
+    def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+        test = pd.DataFrame({'city': ['chicago', 'los angeles']})
+
+        enc = encoders.BaseNEncoder(handle_missing='value', handle_unknown='return_nan')
+        enc.fit(train)
+        result = enc.transform(test)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(train, original)
+
+    def test_inverse_transform_HaveHandleMissingValueAndHandleUnknownReturnNan_ExpectBestInverse(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+        test = pd.DataFrame({'city': ['chicago', np.nan, 'los angeles']})
+        expected = pd.DataFrame({'city': ['chicago', np.nan, np.nan]})
+
+        enc = encoders.BaseNEncoder(handle_missing='value', handle_unknown='return_nan')
+        enc.fit(train)
+        result = enc.transform(test)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(expected, original)

category_encoders/tests/test_one_hot.py

@@ -1,7 +1,7 @@
 import pandas as pd
 from unittest import TestCase  # or `from unittest import ...` if on Python 3.4+
 import numpy as np
-
+import warnings
 import category_encoders.tests.test_utils as tu
 
 import category_encoders as encoders
@@ -201,3 +201,59 @@ def test_HandleUnknown_HaveOnlyKnown_ExpectSecondColumn(self):
         expected = [[1, 0, 0],
                     [0, 1, 0]]
         self.assertEqual(result.values.tolist(), expected)
+
+    def test_inverse_transform_HaveNanInTrainAndHandleMissingValue_ExpectReturnedWithNan(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+
+        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='value')
+        result = enc.fit_transform(train)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(train, original)
+
+    def test_inverse_transform_HaveNanInTrainAndHandleMissingReturnNan_ExpectReturnedWithNan(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+
+        enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='value')
+        result = enc.fit_transform(train)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(train, original)
+
+    def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+        test = pd.DataFrame({'city': ['chicago', 'los angeles']})
+
+        enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='return_nan')
+        enc.fit(train)
+        result = enc.transform(test)
+
+        with warnings.catch_warnings(record=True) as w:
+            enc.inverse_transform(result)
+
+            self.assertEqual(1, len(w))
+            self.assertEqual('inverse_transform is not supported because transform impute '
+                             'the unknown category nan when encode city', str(w[0].message))
+
+    def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+        test = pd.DataFrame({'city': ['chicago', 'los angeles']})
+
+        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='return_nan')
+        enc.fit(train)
+        result = enc.transform(test)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(train, original)
+
+    def test_inverse_transform_HaveHandleMissingValueAndHandleUnknownReturnNan_ExpectBestInverse(self):
+        train = pd.DataFrame({'city': ['chicago', np.nan]})
+        test = pd.DataFrame({'city': ['chicago', np.nan, 'los angeles']})
+        expected = pd.DataFrame({'city': ['chicago', np.nan, np.nan]})
+
+        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='return_nan')
+        enc.fit(train)
+        result = enc.transform(test)
+        original = enc.inverse_transform(result)
+
+        pd.testing.assert_frame_equal(expected, original)