API: define _constructor_expanddim for subclassing Series and DataFrame

Author: sinhrks

doc/source/faq.rst

@@ -369,3 +369,151 @@ just a thin layer around the ``QTableView``.
 	    mw = MainWidget()
 	    mw.show()
 	    app.exec_()
+
+
+
+.. _ref-subclassing-pandas:
+
+Subclassing pandas Data Structures
+----------------------------------
+
+This section describes how to subclass ``pandas`` data structures to meet more specific needs. There are 2 points to be cared:
+
+1. Override constructor properties.
+2. Define original properties
+
+.. note:: You can find actual example in `geopandas <https://github.com/geopandas/geopandas>`_ project.
+
+Override Constructor Properties
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each data structures have constructor properties to specifying data constructors. By overriding these properties, you can retain defined-classes through ``pandas`` data manipulations.
+
+There are 3 constructors to be defined:
+
+- ``_constructor``: Used when a manipulation result has the same dimensionalities as the original.
+- ``_constructor_sliced``: Used when a manipulation result has the lower dimensionalities as the original, such as ``DataFrame`` single columns slicing.
+- ``_constructor_expanddim``: Used when a manipulation result has the higher dimensionalities as the original, such as ``Series.to_frame()`` and ``DataFrame.to_panel()``.
+
+Following table shows how ``pandas`` data structures define constructor properties by default.
+
+===========================  ======================= =================== =======================
+Property Attributes          ``Series``              ``DataFrame``       ``Panel``
+===========================  ======================= =================== =======================
+``_constructor``             ``Series``              ``DataFrame``       ``Panel``
+``_constructor_sliced``      ``NotImplementedError`` ``Series``          ``DataFrame``
+``_constructor_expanddim``   ``DataFrame``           ``Panel``           ``NotImplementedError``
+===========================  ======================= =================== =======================
+
+Below example shows how to define ``SubclassedSeries`` and ``SubclassedDataFrame`` overriding constructor properties.
+
+.. code-block:: python
+
+   class SubclassedSeries(Series):
+
+       @property
+       def _constructor(self):
+           return SubclassedSeries
+
+       @property
+       def _constructor_expanddim(self):
+           return SubclassedDataFrame
+
+   class SubclassedDataFrame(DataFrame):
+
+       @property
+       def _constructor(self):
+           return SubclassedDataFrame
+
+       @property
+       def _constructor_sliced(self):
+           return SubclassedSeries
+
+.. code-block:: python
+
+   >>> s = SubclassedSeries([1, 2, 3])
+   >>> type(s)
+   <class '__main__.SubclassedSeries'>
+
+   >>> to_framed = s.to_frame()
+   >>> type(to_framed)
+   <class '__main__.SubclassedDataFrame'>
+
+   >>> df = SubclassedDataFrame({'A', [1, 2, 3], 'B': [4, 5, 6], 'C': [7, 8, 9]})
+   >>> df
+      A  B  C
+   0  1  4  7
+   1  2  5  8
+   2  3  6  9
+
+   >>> type(df)
+   <class '__main__.SubclassedDataFrame'>
+
+   >>> sliced1 = df[['A', 'B']]
+   >>> sliced1
+      A  B
+   0  1  4
+   1  2  5
+   2  3  6
+   >>> type(sliced1)
+   <class '__main__.SubclassedDataFrame'>
+
+   >>> sliced2 = df['A']
+   >>> sliced2
+   0    1
+   1    2
+   2    3
+   Name: A, dtype: int64
+   >>> type(sliced2)
+   <class '__main__.SubclassedSeries'>
+
+Define Original Properties
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To let original data structures have additional properties, you should let ``pandas`` knows what properties are added. It is because ``pandas`` maps unknown properties to data names overriding ``__getattribute__``. Defining original properties can be done either ways:
+
+1. Define ``_internal_names`` and ``_internal_names_set`` for temporary properties which WILL NOT be passed to manipulation results.
+2. Define ``_metadata`` for normal properties which will be passed to manipulation results.
+
+Below is an example to define 2 original properties, "internal_cache" as a temporary property and "added_property" as a normal property
+
+.. code-block:: python
+
+   class SubclassedDataFrame2(DataFrame):
+
+       # temporary properties
+       _internal_names = DataFrame._internal_names + ['internal_cache']
+       _internal_names_set = set(_internal_names)
+
+       # normal properties
+       _metadata = ['added_property']
+
+       @property
+       def _constructor(self):
+           return SubclassedDataFrame2
+
+.. code-block:: python
+
+   >>> df = SubclassedDataFrame2({'A', [1, 2, 3], 'B': [4, 5, 6], 'C': [7, 8, 9]})
+   >>> df
+      A  B  C
+   0  1  4  7
+   1  2  5  8
+   2  3  6  9
+
+   >>> df.internal_cache = 'cached'
+   >>> df.added_property = 'property'
+
+   >>> df.internal_cache
+   cached
+   >>> df.added_property
+   property
+
+   # properties defined in _internal_names is reset after manipulation
+   >>> df[['A', 'B']].internal_cache
+   AttributeError: 'SubclassedDataFrame2' object has no attribute 'internal_cache'
+
+   # properties defined in _metadata is retained
+   >>> df[['A', 'B']].added_property
+   property
+

pandas/core/frame.py

@@ -191,6 +191,11 @@ def _constructor(self):
 
     _constructor_sliced = Series
 
+    @property
+    def _constructor_expanddim(self):
+        from pandas.core.panel import Panel
+        return Panel
+
     def __init__(self, data=None, index=None, columns=None, dtype=None,
                  copy=False):
         if data is None:
@@ -1064,8 +1069,6 @@ def to_panel(self):
         -------
         panel : Panel
         """
-        from pandas.core.panel import Panel
-
         # only support this kind for now
         if (not isinstance(self.index, MultiIndex) or  # pragma: no cover
                 len(self.index.levels) != 2):
@@ -1103,7 +1106,7 @@ def to_panel(self):
                                               shape=shape,
                                               ref_items=selfsorted.columns)
 
-        return Panel(new_mgr)
+        return self._constructor_expanddim(new_mgr)
 
     to_wide = deprecate('to_wide', to_panel)
 
@@ -4413,12 +4416,12 @@ def mode(self, axis=0, numeric_only=False):
         """
         Gets the mode(s) of each element along the axis selected. Empty if nothing
         has 2+ occurrences. Adds a row for each mode per label, fills in gaps
-        with nan. 
-        
+        with nan.
+
         Note that there could be multiple values returned for the selected
-        axis (when more than one item share the maximum frequency), which is the 
-        reason why a dataframe is returned. If you want to impute missing values 
-        with the mode in a dataframe ``df``, you can just do this: 
+        axis (when more than one item share the maximum frequency), which is the
+        reason why a dataframe is returned. If you want to impute missing values
+        with the mode in a dataframe ``df``, you can just do this:
         ``df.fillna(df.mode().iloc[0])``
 
         Parameters

pandas/core/generic.py

@@ -154,6 +154,10 @@ def _local_dir(self):
     def _constructor_sliced(self):
         raise NotImplementedError
 
+    @property
+    def _constructor_expanddim(self):
+        raise NotImplementedError
+
     #----------------------------------------------------------------------
     # Axis
 

pandas/core/series.py

@@ -236,6 +236,11 @@ def from_array(cls, arr, index=None, name=None, dtype=None, copy=False,
     def _constructor(self):
         return Series
 
+    @property
+    def _constructor_expanddim(self):
+        from pandas.core.frame import DataFrame
+        return DataFrame
+
     # types
     @property
     def _can_hold_na(self):
@@ -1047,11 +1052,10 @@ def to_frame(self, name=None):
         -------
         data_frame : DataFrame
         """
-        from pandas.core.frame import DataFrame
         if name is None:
-            df = DataFrame(self)
+            df = self._constructor_expanddim(self)
         else:
-            df = DataFrame({name: self})
+            df = self._constructor_expanddim({name: self})
 
         return df
 

pandas/tests/test_frame.py

@@ -31,7 +31,7 @@
 import pandas.core.common as com
 import pandas.core.format as fmt
 import pandas.core.datetools as datetools
-from pandas import (DataFrame, Index, Series, notnull, isnull,
+from pandas import (DataFrame, Index, Series, Panel, notnull, isnull,
                     MultiIndex, DatetimeIndex, Timestamp, date_range,
                     read_csv, timedelta_range, Timedelta,
                     option_context)
@@ -14099,6 +14099,47 @@ def _constructor(self):
         # GH9776
         self.assertEqual(df.iloc[0:1, :].testattr, 'XXX')
 
+    def test_to_panel_expanddim(self):
+
+        class SubclassedFrame(DataFrame):
+            @property
+            def _constructor_expanddim(self):
+                return SubclassedPanel
+
+        class SubclassedPanel(Panel):
+            pass
+
+        index = MultiIndex.from_tuples([(0, 0), (0, 1), (0, 2)])
+        df = SubclassedFrame({'X':[1, 2, 3], 'Y': [4, 5, 6]}, index=index)
+        result = df.to_panel()
+        self.assertTrue(isinstance(result, SubclassedPanel))
+        expected = SubclassedPanel([[[1, 2, 3]], [[4, 5, 6]]],
+                                   items=['X', 'Y'], major_axis=[0],
+                                   minor_axis=[0, 1, 2])
+        tm.assert_panel_equal(result, expected)
+>>>>>>> API: define _constructor_expanddim for subclassing Series and DataFrame
+
+    def test_dataframe_metadata(self):
+
+        class TestDataFrame(DataFrame):
+            _metadata = ['testattr']
+
+            @property
+            def _constructor(self):
+                return TestDataFrame
+
+
+        df = TestDataFrame({'X': [1, 2, 3], 'Y': [1, 2, 3]},
+                           index=['a', 'b', 'c'])
+        df.testattr = 'XXX'
+
+        self.assertEqual(df.testattr, 'XXX')
+        self.assertEqual(df[['X']].testattr, 'XXX')
+        self.assertEqual(df.loc[['a', 'b'], :].testattr, 'XXX')
+        self.assertEqual(df.iloc[[0, 1], :].testattr, 'XXX')
+        # GH9776
+        self.assertEqual(df.iloc[0:1, :].testattr, 'XXX')
+
 
 def skip_if_no_ne(engine='numexpr'):
     if engine == 'numexpr':

pandas/tests/test_series.py

@@ -6754,6 +6754,21 @@ def test_searchsorted_sorter(self):
         e = np.array([0, 2])
         tm.assert_array_equal(r, e)
 
+    def test_to_frame_expanddim(self):
+
+        class SubclassedSeries(Series):
+            @property
+            def _constructor_expanddim(self):
+                return SubclassedFrame
+
+        class SubclassedFrame(DataFrame):
+            pass
+
+        s = SubclassedSeries([1, 2, 3], name='X')
+        result = s.to_frame()
+        self.assertTrue(isinstance(result, SubclassedFrame))
+        expected = SubclassedFrame({'X': [1, 2, 3]})
+        assert_frame_equal(result, expected)
 
 
 class TestSeriesNonUnique(tm.TestCase):