Updating README.rst and fixing features generator bug

Author: devforfu

README.rst

@@ -240,6 +240,7 @@ in a list::
            [ 0.        ],
            [ 1.22474487]])
 
+
 Columns that don't need any transformation
 ******************************************
 
@@ -282,6 +283,59 @@ passing it as the ``default`` argument to the mapper:
 Using ``default=False`` (the default) drops unselected columns. Using
 ``default=None`` pass the unselected columns unchanged.
 
+
+Same transformer for the multiple columns
+*****************************************
+
+Sometimes it is required to apply the same transformation to several dataframe columns.
+To simplify this process, the package provides ``gen_features`` function which accepts a list
+of columns and feature transformer class (or list of classes), and generates a feature definition,
+acceptable by ``DataFrameMapper``.
+
+For example, consider a dataset with three categorical columns, 'col1', 'col2', and 'col3',
+To binarize each of them, one could pass column names and ``LabelBinarizer`` transformer class
+into generator, and then use returned definition as ``features`` argument for ``DataFrameMapper``:
+
+    >>> from sklearn_pandas import gen_features
+    >>> feature_def = gen_features(
+    ...     columns=['col1', 'col2', 'col3'],
+    ...     classes=[sklearn.preprocessing.LabelEncoder]
+    ... )
+    >>> feature_def
+    [('col1', [LabelEncoder()]), ('col2', [LabelEncoder()]), ('col3', [LabelEncoder()])]
+    >>> mapper5 = DataFrameMapper(feature_def)
+    >>> data5 = pd.DataFrame({
+    ...     'col1': ['yes', 'no', 'yes'],
+    ...     'col2': [True, False, False],
+    ...     'col3': ['one', 'two', 'three']
+    ... })
+    >>> mapper5.fit_transform(data5)
+    array([[1, 1, 0],
+           [0, 0, 2],
+           [1, 0, 1]])
+
+If it is required to override some of transformer parameters, then a dict with 'class' key and
+transformer parameters should be provided. For example, consider a dataset with missing values.
+Then the following code could be used to override default imputing strategy:
+
+    >>> feature_def = gen_features(
+    ...     columns=[['col1'], ['col2'], ['col3']],
+    ...     classes=[{'class': sklearn.preprocessing.Imputer, 'strategy': 'most_frequent'}]
+    ... )
+    >>> mapper6 = DataFrameMapper(feature_def)
+    >>> data6 = pd.DataFrame({
+    ...     'col1': [None, 1, 1, 2, 3],
+    ...     'col2': [True, False, None, None, True],
+    ...     'col3': [0, 0, 0, None, None]
+    ... })
+    >>> mapper6.fit_transform(data6)
+    array([[ 1.,  1.,  0.],
+           [ 1.,  0.,  0.],
+           [ 1.,  1.,  0.],
+           [ 2.,  1.,  0.],
+           [ 3.,  1.,  0.]])
+
+
 Feature selection and other supervised transformations
 ******************************************************
 

sklearn_pandas/features_generator.py

@@ -6,17 +6,17 @@ def gen_features(columns, classes=None):
 
     columns     a list of column names to generate features for.
 
-    classes     a list of classes for each feature, a list dictionaries with
+    classes     a list of classes for each feature, a list of dictionaries with
                 transformer class and init parameters, or None.
 
                 If list of classes is provided, then each of them is
-                instantiated with default arguments:
+                instantiated with default arguments. Example:
 
                     classes = [StandardScaler, LabelBinarizer]
 
                 If list of dictionaries is provided, then each of them should
                 have a 'class' key with transformer class. All other keys are
-                passed into 'class' value constructor:
+                passed into 'class' value constructor. Example:
 
                     classes = [
                         {'class': StandardScaler, 'with_mean': False},
@@ -34,17 +34,22 @@ def gen_features(columns, classes=None):
     for column in columns:
         feature_transformers = []
 
-        for definition in classes:
-            if isinstance(definition, dict):
-                params = definition.copy()
-                klass = params.pop('class')
-                feature_transformers.append(klass(**params))
-            elif isinstance(definition, type):
-                feature_transformers.append(definition())
+        classes = [cls for cls in classes if cls is not None]
+        if not classes:
+            feature_defs.append((column, None))
 
-        if not feature_transformers:
-            feature_transformers = None
+        else:
+            for definition in classes:
+                if isinstance(definition, dict):
+                    params = definition.copy()
+                    klass = params.pop('class')
+                    feature_transformers.append(klass(**params))
+                else:
+                    feature_transformers.append(definition())
 
-        feature_defs.append((column, feature_transformers))
+            if not feature_transformers:
+                feature_transformers = None
+
+            feature_defs.append((column, feature_transformers))
 
     return feature_defs

tests/test_features_generator.py

@@ -39,17 +39,18 @@ def simple_dataset():
     })
 
 
-@pytest.mark.parametrize('columns', [['colA', 'colB', 'colC']])
-def test_generate_features_with_default_parameters(columns):
+def test_generate_features_with_default_parameters():
     """
     Tests generating features from classes with default init arguments.
     """
+    columns = ['colA', 'colB', 'colC']
     feature_defs = gen_features(columns=columns, classes=[MockClass])
     assert len(feature_defs) == len(columns)
 
     feature_dict = dict(feature_defs)
-    assert columns == sorted(feature_dict)
+    assert columns == sorted(feature_dict.keys())
 
+    # default init arguments for MockClass for clarification.
     expected = {'value': 1, 'name': 'class'}
     for column, transformers in feature_dict.items():
         for obj in transformers:
@@ -75,9 +76,9 @@ def test_generate_features_with_several_classes():
         assert_attributes(transformers[2], name='mockB', value=None)
 
 
-def test_generate_features_with_none_transformers():
+def test_generate_features_with_none_only_transformers():
     """
-    Tests generating "dummy" feature definiton which doesn't apply any
+    Tests generating "dummy" feature definition which doesn't apply any
     transformation.
     """
     feature_defs = gen_features(