From d932705d753145608429f011ca29f8c3114a9f27 Mon Sep 17 00:00:00 2001 From: Brock Date: Fri, 14 Aug 2020 14:41:26 -0700 Subject: [PATCH 1/5] REF: move towards making _apply_blockwise actually block-wise --- pandas/core/window/rolling.py | 56 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 966773b7c6982..3ff83c2ab1f09 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -6,7 +6,17 @@ from functools import partial import inspect from textwrap import dedent -from typing import Callable, Dict, List, Optional, Set, Tuple, Type, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + Union, +) import numpy as np @@ -55,6 +65,9 @@ ) from pandas.core.window.numba_ import generate_numba_apply_func +if TYPE_CHECKING: + from pandas import Series + def calculate_center_offset(window) -> int: """ @@ -394,12 +407,16 @@ def _wrap_results(self, results, blocks, obj, exclude=None) -> FrameOrSeries: """ from pandas import Series, concat + if obj.ndim == 1: + if not results: + raise DataError("No numeric types to aggregate") + assert len(results) == 1 + return Series(results[0], index=obj.index, name=obj.name) + final = [] for result, block in zip(results, blocks): - result = self._wrap_result(result, block=block, obj=obj) - if result.ndim == 1: - return result + result = type(obj)(result, index=obj.index, columns=block.columns) final.append(result) # if we have an 'on' column @@ -487,6 +504,23 @@ def _get_window_indexer(self, window: int) -> BaseIndexer: return VariableWindowIndexer(index_array=self._on.asi8, window_size=window) return FixedWindowIndexer(window_size=window) + def _apply_series(self, homogeneous_func: Callable[..., ArrayLike]) -> "Series": + """ + Series version of _apply_blockwise + """ + from pandas import Series # TODO: use _constructor? + + _, obj = self._create_blocks(self._selected_obj) + values = obj.values + + try: + values = self._prep_values(obj.values) + except (TypeError, NotImplementedError) as err: + raise DataError("No numeric types to aggregate") from err + + result = homogeneous_func(values) + return Series(result, index=obj.index, name=obj.name) + def _apply_blockwise( self, homogeneous_func: Callable[..., ArrayLike] ) -> FrameOrSeries: @@ -494,6 +528,9 @@ def _apply_blockwise( Apply the given function to the DataFrame broken down into homogeneous sub-frames. """ + if self._selected_obj.ndim == 1: + return self._apply_series(homogeneous_func) + # This isn't quite blockwise, since `blocks` is actually a collection # of homogenenous DataFrames. blocks, obj = self._create_blocks(self._selected_obj) @@ -505,13 +542,10 @@ def _apply_blockwise( try: values = self._prep_values(b.values) - except (TypeError, NotImplementedError) as err: - if isinstance(obj, ABCDataFrame): - skipped.append(i) - exclude.extend(b.columns) - continue - else: - raise DataError("No numeric types to aggregate") from err + except (TypeError, NotImplementedError): + skipped.append(i) + exclude.extend(b.columns) + continue result = homogeneous_func(values) results.append(result) From e58a5a2d26ee96a221a13d61a23aefaeca796bcf Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 17 Aug 2020 08:53:05 -0700 Subject: [PATCH 2/5] mypy fixup --- pandas/core/window/rolling.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 3ff83c2ab1f09..faa5cd350f986 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -6,17 +6,7 @@ from functools import partial import inspect from textwrap import dedent -from typing import ( - TYPE_CHECKING, - Callable, - Dict, - List, - Optional, - Set, - Tuple, - Type, - Union, -) +from typing import Callable, Dict, List, Optional, Set, Tuple, Type, Union import numpy as np @@ -65,9 +55,6 @@ ) from pandas.core.window.numba_ import generate_numba_apply_func -if TYPE_CHECKING: - from pandas import Series - def calculate_center_offset(window) -> int: """ @@ -504,7 +491,9 @@ def _get_window_indexer(self, window: int) -> BaseIndexer: return VariableWindowIndexer(index_array=self._on.asi8, window_size=window) return FixedWindowIndexer(window_size=window) - def _apply_series(self, homogeneous_func: Callable[..., ArrayLike]) -> "Series": + def _apply_series( + self, homogeneous_func: Callable[..., ArrayLike] + ) -> FrameOrSeries: """ Series version of _apply_blockwise """ @@ -2311,6 +2300,7 @@ def _get_window_indexer(self, window: int) -> GroupbyRollingIndexer: if isinstance(self.window, BaseIndexer): rolling_indexer = type(self.window) indexer_kwargs = self.window.__dict__ + assert isinstance(indexer_kwargs, dict) # for mypy # We'll be using the index of each group later indexer_kwargs.pop("index_array", None) elif self.is_freq_type: From 0582f348bca36272b135519738524b4bb64def58 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 17 Aug 2020 09:16:58 -0700 Subject: [PATCH 3/5] mypy fixup --- pandas/core/window/rolling.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index faa5cd350f986..a1c881c38b6b5 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -6,13 +6,23 @@ from functools import partial import inspect from textwrap import dedent -from typing import Callable, Dict, List, Optional, Set, Tuple, Type, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + Union, +) import numpy as np from pandas._libs.tslibs import BaseOffset, to_offset import pandas._libs.window.aggregations as window_aggregations -from pandas._typing import ArrayLike, Axis, FrameOrSeries, Scalar +from pandas._typing import ArrayLike, Axis, FrameOrSeriesUnion, Scalar from pandas.compat._optional import import_optional_dependency from pandas.compat.numpy import function as nv from pandas.util._decorators import Appender, Substitution, cache_readonly, doc @@ -55,6 +65,9 @@ ) from pandas.core.window.numba_ import generate_numba_apply_func +if TYPE_CHECKING: + from pandas import Series + def calculate_center_offset(window) -> int: """ @@ -145,7 +158,7 @@ class _Window(PandasObject, ShallowMixin, SelectionMixin): def __init__( self, - obj: FrameOrSeries, + obj: FrameOrSeriesUnion, window=None, min_periods: Optional[int] = None, center: bool = False, @@ -219,7 +232,7 @@ def _validate_get_window_bounds_signature(window: BaseIndexer) -> None: f"get_window_bounds" ) - def _create_blocks(self, obj: FrameOrSeries): + def _create_blocks(self, obj: FrameOrSeriesUnion): """ Split data into blocks & return conformed data. """ @@ -381,7 +394,7 @@ def _wrap_result(self, result, block=None, obj=None): return type(obj)(result, index=index, columns=block.columns) return result - def _wrap_results(self, results, blocks, obj, exclude=None) -> FrameOrSeries: + def _wrap_results(self, results, blocks, obj, exclude=None) -> FrameOrSeriesUnion: """ Wrap the results. @@ -491,9 +504,7 @@ def _get_window_indexer(self, window: int) -> BaseIndexer: return VariableWindowIndexer(index_array=self._on.asi8, window_size=window) return FixedWindowIndexer(window_size=window) - def _apply_series( - self, homogeneous_func: Callable[..., ArrayLike] - ) -> FrameOrSeries: + def _apply_series(self, homogeneous_func: Callable[..., ArrayLike]) -> "Series": """ Series version of _apply_blockwise """ @@ -512,7 +523,7 @@ def _apply_series( def _apply_blockwise( self, homogeneous_func: Callable[..., ArrayLike] - ) -> FrameOrSeries: + ) -> FrameOrSeriesUnion: """ Apply the given function to the DataFrame broken down into homogeneous sub-frames. @@ -2259,7 +2270,7 @@ def _apply( def _constructor(self): return Rolling - def _create_blocks(self, obj: FrameOrSeries): + def _create_blocks(self, obj: FrameOrSeriesUnion): """ Split data into blocks & return conformed data. """ From e383481ca674ca1cefb6e3290d64de769dcfd433 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 18 Aug 2020 08:49:07 -0700 Subject: [PATCH 4/5] Series->_constructor --- pandas/core/window/rolling.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index a1c881c38b6b5..a302987a4416a 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -508,8 +508,6 @@ def _apply_series(self, homogeneous_func: Callable[..., ArrayLike]) -> "Series": """ Series version of _apply_blockwise """ - from pandas import Series # TODO: use _constructor? - _, obj = self._create_blocks(self._selected_obj) values = obj.values @@ -519,7 +517,7 @@ def _apply_series(self, homogeneous_func: Callable[..., ArrayLike]) -> "Series": raise DataError("No numeric types to aggregate") from err result = homogeneous_func(values) - return Series(result, index=obj.index, name=obj.name) + return obj._constructor(result, index=obj.index, name=obj.name) def _apply_blockwise( self, homogeneous_func: Callable[..., ArrayLike] From 8b93dab8562db050d0f31d069fe7a1403eac3f82 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 22 Aug 2020 15:00:23 -0700 Subject: [PATCH 5/5] dummy commit to force CI