PERF: remove use of Panel & perf in rolling corr/cov

closes pandas-dev#17917

Author: jreback

asv_bench/benchmarks/rolling.py

@@ -11,7 +11,7 @@ class Methods(object):
               [10, 1000],
               ['int', 'float'],
               ['median', 'mean', 'max', 'min', 'std', 'count', 'skew', 'kurt',
-               'sum', 'corr', 'cov'])
+               'sum'])
     param_names = ['contructor', 'window', 'dtype', 'method']
 
     def setup(self, contructor, window, dtype, method):
@@ -23,6 +23,27 @@ def time_rolling(self, contructor, window, dtype, method):
         getattr(self.roll, method)()
 
 
+class Pairwise(object):
+
+    sample_time = 0.2
+    params = ([10, 1000, None],
+              ['corr', 'cov'],
+              [True, False])
+    param_names = ['window', 'method', 'pairwise']
+
+    def setup(self, window, method, pairwise):
+        N = 10**4
+        arr = np.random.random(N)
+        self.df = pd.DataFrame(arr)
+
+    def time_pairwise(self, window, method, pairwise):
+        if window is None:
+            r = self.df.expanding()
+        else:
+            r = self.df.rolling(window=window)
+        getattr(r, method)(self.df, pairwise=pairwise)
+
+
 class Quantile(object):
 
     sample_time = 0.2

pandas/core/indexes/base.py

@@ -1108,6 +1108,11 @@ def _assert_can_do_op(self, value):
     def nlevels(self):
         return 1
 
+    @property
+    def levels(self):
+        """ return a list my levels """
+        return [list(self)]
+
     def _get_names(self):
         return FrozenList((self.name, ))
 

pandas/core/window.py

@@ -1863,25 +1863,38 @@ def dataframe_from_int_dict(data, frame_template):
                             results[i][j] = f(*_prep_binary(arg1.iloc[:, i],
                                                             arg2.iloc[:, j]))
 
-                # TODO: not the most efficient (perf-wise)
-                # though not bad code-wise
-                from pandas import Panel, MultiIndex, concat
-
-                with warnings.catch_warnings(record=True):
-                    p = Panel.from_dict(results).swapaxes('items', 'major')
-                    if len(p.major_axis) > 0:
-                        p.major_axis = arg1.columns[p.major_axis]
-                    if len(p.minor_axis) > 0:
-                        p.minor_axis = arg2.columns[p.minor_axis]
-
-                if len(p.items):
+                from pandas import MultiIndex, concat
+
+                result_index = arg1.index.union(arg2.index)
+                if len(result_index):
+
+                    # construct result frame
                     result = concat(
-                        [p.iloc[i].T for i in range(len(p.items))],
-                        keys=p.items)
+                        [concat([results[i][j]
+                                 for j, c in enumerate(arg2.columns)],
+                                ignore_index=True)
+                         for i, c in enumerate(arg1.columns)],
+                        ignore_index=True,
+                        axis=1)
+                    result.columns = arg1.columns
+
+                    # set the index and reorder
+                    if arg2.columns.nlevels > 1:
+                        result.index = MultiIndex.from_product(
+                            arg2.columns.levels + result_index.levels)
+                        result = result.reorder_levels([2, 0, 1]).sort_index()
+                    else:
+                        result.index = MultiIndex.from_product(
+                            [range(len(arg2.columns)),
+                             range(len(result_index))])
+                        result = result.swaplevel(1, 0).sort_index()
+                        result.index = MultiIndex.from_product(
+                            result_index.levels + arg2.columns.levels)
                 else:
 
+                    # empty result
                     result = DataFrame(
-                        index=MultiIndex(levels=[arg1.index, arg1.columns],
+                        index=MultiIndex(levels=[arg1.index, arg2.columns],
                                          labels=[[], []]),
                         columns=arg2.columns,
                         dtype='float64')
@@ -1890,9 +1903,9 @@ def dataframe_from_int_dict(data, frame_template):
                 # reset our column names to arg2 names
                 # careful not to mutate the original names
                 result.columns = result.columns.set_names(
-                    arg2.columns.names)
+                    arg1.columns.names)
                 result.index = result.index.set_names(
-                    arg1.index.names + arg1.columns.names)
+                    result_index.names + arg2.columns.names)
 
                 return result
 

pandas/tests/test_window.py

@@ -16,6 +16,7 @@
 import pandas.tseries.offsets as offsets
 from pandas.core.base import SpecificationError
 from pandas.errors import UnsupportedFunctionCall
+from pandas.core.sorting import safe_sort
 import pandas.util.testing as tm
 import pandas.util._test_decorators as td
 from pandas.compat import range, zip, PY3
@@ -1814,7 +1815,7 @@ def compare(self, result, expected):
         result = result.dropna().values
         expected = expected.dropna().values
 
-        tm.assert_numpy_array_equal(result, expected)
+        tm.assert_numpy_array_equal(result, expected, check_dtype=False)
 
     @pytest.mark.parametrize('f', [lambda x: x.cov(), lambda x: x.corr()])
     def test_no_flex(self, f):
@@ -1839,15 +1840,19 @@ def test_no_flex(self, f):
     def test_pairwise_with_self(self, f):
 
         # DataFrame with itself, pairwise=True
-        results = [f(df) for df in self.df1s]
-        for (df, result) in zip(self.df1s, results):
+        # note that we may construct the 1st level of the MI
+        # in a non-motononic way, so compare accordingly
+        results = []
+        for i, df in enumerate(self.df1s):
+            result = f(df)
             tm.assert_index_equal(result.index.levels[0],
                                   df.index,
                                   check_names=False)
-            tm.assert_index_equal(result.index.levels[1],
-                                  df.columns,
-                                  check_names=False)
+            tm.assert_numpy_array_equal(safe_sort(result.index.levels[1]),
+                                        safe_sort(df.columns.unique()))
             tm.assert_index_equal(result.columns, df.columns)
+            results.append(df)
+
         for i, result in enumerate(results):
             if i > 0:
                 self.compare(result, results[0])
@@ -1885,9 +1890,8 @@ def test_pairwise_with_other(self, f):
             tm.assert_index_equal(result.index.levels[0],
                                   df.index,
                                   check_names=False)
-            tm.assert_index_equal(result.index.levels[1],
-                                  self.df2.columns,
-                                  check_names=False)
+            tm.assert_numpy_array_equal(safe_sort(result.index.levels[1]),
+                                        safe_sort(self.df2.columns.unique()))
         for i, result in enumerate(results):
             if i > 0:
                 self.compare(result, results[0])