BUG: int dtype for get_dummies

closes #8725    Changes `get_dummies` return columns with `uint8`
dtypes instead of coercing to floats if they were alongside other
float columns.

Author: Tom Augspurger <[email protected]>

Closes #13796 from TomAugspurger/get_dummies_dtype and squashes the following commits:

cace0f7 [Tom Augspurger] BUG: int dtype for get_dummies

Author: TomAugspurger

doc/source/whatsnew/v0.19.0.txt

@@ -371,6 +371,32 @@ Previous versions of pandas would permanently silence numpy's ufunc error handli
 
 After upgrading pandas, you may see *new* ``RuntimeWarnings`` being issued from your code. These are likely legitimate, and the underlying cause likely existed in the code when using previous versions of pandas that simply silenced the warning. Use `numpy.errstate <http://docs.scipy.org/doc/numpy/reference/generated/numpy.errstate.html>`__ around the source of the ``RuntimeWarning`` to control how these conditions are handled.
 
+.. _whatsnew_0190.get_dummies_dtypes:
+
+get_dummies dtypes
+^^^^^^^^^^^^^^^^^^
+
+The ``pd.get_dummies`` function now returns dummy-encoded columns as small integers, rather than floats (:issue:`8725`)
+
+Previous behaviour:
+
+.. code-block:: ipython
+
+   In [1]:    pd.get_dummies(['a', 'b', 'a', 'c']).dtypes
+
+   Out[1]:
+   a    float64
+   b    float64
+   c    float64
+   dtype: object
+
+New Behavior:
+
+.. ipython:: python
+
+   pd.get_dummies(['a', 'b', 'a', 'c']).dtypes
+
+
 .. _whatsnew_0190.enhancements.other:
 
 Other enhancements
@@ -479,7 +505,6 @@ API changes
 - ``Series.unique()`` with datetime and timezone now returns return array of ``Timestamp`` with timezone (:issue:`13565`)
 
 
-
 .. _whatsnew_0190.api.tolist:
 
 ``Series.tolist()`` will now return Python types
@@ -1355,7 +1380,7 @@ Bug Fixes
 - Bug in using NumPy ufunc with ``PeriodIndex`` to add or subtract integer raise ``IncompatibleFrequency``. Note that using standard operator like ``+`` or ``-`` is recommended, because standard operators use more efficient path (:issue:`13980`)
 - Bug in operations on ``NaT`` returning ``float`` instead of ``datetime64[ns]`` (:issue:`12941`)
 - Bug in ``Series`` flexible arithmetic methods (like ``.add()``) raises ``ValueError`` when ``axis=None`` (:issue:`13894`)
-- Bug in ``DataFrame.to_csv()`` with ``MultiIndex`` columns in which a stray empty line was added (:issue:`6618`) 
+- Bug in ``DataFrame.to_csv()`` with ``MultiIndex`` columns in which a stray empty line was added (:issue:`6618`)
 
 
 - Bug in ``Index`` raises ``KeyError`` displaying incorrect column when column is not in the df and columns contains duplicate values (:issue:`13822`)

pandas/core/reshape.py

@@ -1161,14 +1161,17 @@ def get_empty_Frame(data, sparse):
             sp_indices = sp_indices[1:]
             dummy_cols = dummy_cols[1:]
         for col, ixs in zip(dummy_cols, sp_indices):
-            sarr = SparseArray(np.ones(len(ixs)),
-                               sparse_index=IntIndex(N, ixs), fill_value=0)
+            sarr = SparseArray(np.ones(len(ixs), dtype=np.uint8),
+                               sparse_index=IntIndex(N, ixs), fill_value=0,
+                               dtype=np.uint8)
             sparse_series[col] = SparseSeries(data=sarr, index=index)
 
-        return SparseDataFrame(sparse_series, index=index, columns=dummy_cols)
+        out = SparseDataFrame(sparse_series, index=index, columns=dummy_cols,
+                              dtype=np.uint8)
+        return out
 
     else:
-        dummy_mat = np.eye(number_of_cols).take(codes, axis=0)
+        dummy_mat = np.eye(number_of_cols, dtype=np.uint8).take(codes, axis=0)
 
         if not dummy_na:
             # reset NaN GH4446

pandas/stats/tests/test_ols.py

@@ -645,6 +645,7 @@ def testWithXEffects(self):
         exp_x = DataFrame([[0., 0., 14., 1.], [0, 1, 17, 1], [1, 0, 48, 1]],
                           columns=['x1_30', 'x1_9', 'x2', 'intercept'],
                           index=res.index, dtype=float)
+        exp_x[['x1_30', 'x1_9']] = exp_x[['x1_30', 'x1_9']].astype(np.uint8)
         assert_frame_equal(res, exp_x.reindex(columns=res.columns))
 
     def testWithXEffectsAndDroppedDummies(self):
@@ -659,6 +660,7 @@ def testWithXEffectsAndDroppedDummies(self):
         exp_x = DataFrame([[1., 0., 14., 1.], [0, 1, 17, 1], [0, 0, 48, 1]],
                           columns=['x1_6', 'x1_9', 'x2', 'intercept'],
                           index=res.index, dtype=float)
+        exp_x[['x1_6', 'x1_9']] = exp_x[['x1_6', 'x1_9']].astype(np.uint8)
 
         assert_frame_equal(res, exp_x.reindex(columns=res.columns))
 

pandas/tests/test_panel.py

@@ -2429,18 +2429,18 @@ def test_truncate(self):
     def test_axis_dummies(self):
         from pandas.core.reshape import make_axis_dummies
 
-        minor_dummies = make_axis_dummies(self.panel, 'minor')
+        minor_dummies = make_axis_dummies(self.panel, 'minor').astype(np.uint8)
         self.assertEqual(len(minor_dummies.columns),
                          len(self.panel.index.levels[1]))
 
-        major_dummies = make_axis_dummies(self.panel, 'major')
+        major_dummies = make_axis_dummies(self.panel, 'major').astype(np.uint8)
         self.assertEqual(len(major_dummies.columns),
                          len(self.panel.index.levels[0]))
 
         mapping = {'A': 'one', 'B': 'one', 'C': 'two', 'D': 'two'}
 
         transformed = make_axis_dummies(self.panel, 'minor',
-                                        transform=mapping.get)
+                                        transform=mapping.get).astype(np.uint8)
         self.assertEqual(len(transformed.columns), 2)
         self.assert_index_equal(transformed.columns, Index(['one', 'two']))
 
@@ -2450,7 +2450,7 @@ def test_get_dummies(self):
         from pandas.core.reshape import get_dummies, make_axis_dummies
 
         self.panel['Label'] = self.panel.index.labels[1]
-        minor_dummies = make_axis_dummies(self.panel, 'minor')
+        minor_dummies = make_axis_dummies(self.panel, 'minor').astype(np.uint8)
         dummies = get_dummies(self.panel['Label'])
         self.assert_numpy_array_equal(dummies.values, minor_dummies.values)