From b1ba512003573ee6b19251139d709ac57a9d2f9a Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Wed, 6 May 2020 15:39:35 +0100 Subject: [PATCH] Backport PR #33693 on branch 1.0.x (BUG: Fix memory issues in rolling.min/max) --- asv_bench/benchmarks/rolling.py | 25 ++++++++++++------------- doc/source/whatsnew/v1.0.4.rst | 2 +- pandas/_libs/window/aggregations.pyx | 16 +++++++--------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py index 7a72622fd5fe3..331086b773eea 100644 --- a/asv_bench/benchmarks/rolling.py +++ b/asv_bench/benchmarks/rolling.py @@ -129,19 +129,18 @@ def time_quantile(self, constructor, window, dtype, percentile, interpolation): self.roll.quantile(percentile, interpolation=interpolation) -class PeakMemFixed: - def setup(self): - N = 10 - arr = 100 * np.random.random(N) - self.roll = pd.Series(arr).rolling(10) - - def peakmem_fixed(self): - # GH 25926 - # This is to detect memory leaks in rolling operations. - # To save time this is only ran on one method. - # 6000 iterations is enough for most types of leaks to be detected - for x in range(6000): - self.roll.max() +class PeakMemFixedWindowMinMax: + + params = ["min", "max"] + + def setup(self, operation): + N = int(1e6) + arr = np.random.random(N) + self.roll = pd.Series(arr).rolling(2) + + def peakmem_fixed(self, operation): + for x in range(5): + getattr(self.roll, operation)() from .pandas_vb_common import setup # noqa: F401 isort:skip diff --git a/doc/source/whatsnew/v1.0.4.rst b/doc/source/whatsnew/v1.0.4.rst index 80f8976826bc8..219f3459aa09c 100644 --- a/doc/source/whatsnew/v1.0.4.rst +++ b/doc/source/whatsnew/v1.0.4.rst @@ -27,7 +27,7 @@ Fixed regressions Bug fixes ~~~~~~~~~ -- +- Bug in :meth:`Rolling.min` and :meth:`Rolling.max`: Growing memory usage after multiple calls when using a fixed window (:issue:`30726`) - Contributors diff --git a/pandas/_libs/window/aggregations.pyx b/pandas/_libs/window/aggregations.pyx index 495b436030120..e998c7e31bd49 100644 --- a/pandas/_libs/window/aggregations.pyx +++ b/pandas/_libs/window/aggregations.pyx @@ -965,8 +965,8 @@ cdef inline numeric calc_mm(int64_t minp, Py_ssize_t nobs, return result -def roll_max_fixed(ndarray[float64_t] values, ndarray[int64_t] start, - ndarray[int64_t] end, int64_t minp, int64_t win): +def roll_max_fixed(float64_t[:] values, int64_t[:] start, + int64_t[:] end, int64_t minp, int64_t win): """ Moving max of 1d array of any numeric type along axis=0 ignoring NaNs. @@ -982,7 +982,7 @@ def roll_max_fixed(ndarray[float64_t] values, ndarray[int64_t] start, make the interval closed on the right, left, both or neither endpoints """ - return _roll_min_max_fixed(values, start, end, minp, win, is_max=1) + return _roll_min_max_fixed(values, minp, win, is_max=1) def roll_max_variable(ndarray[float64_t] values, ndarray[int64_t] start, @@ -1005,8 +1005,8 @@ def roll_max_variable(ndarray[float64_t] values, ndarray[int64_t] start, return _roll_min_max_variable(values, start, end, minp, is_max=1) -def roll_min_fixed(ndarray[float64_t] values, ndarray[int64_t] start, - ndarray[int64_t] end, int64_t minp, int64_t win): +def roll_min_fixed(float64_t[:] values, int64_t[:] start, + int64_t[:] end, int64_t minp, int64_t win): """ Moving min of 1d array of any numeric type along axis=0 ignoring NaNs. @@ -1019,7 +1019,7 @@ def roll_min_fixed(ndarray[float64_t] values, ndarray[int64_t] start, index : ndarray, optional index for window computation """ - return _roll_min_max_fixed(values, start, end, minp, win, is_max=0) + return _roll_min_max_fixed(values, minp, win, is_max=0) def roll_min_variable(ndarray[float64_t] values, ndarray[int64_t] start, @@ -1121,9 +1121,7 @@ cdef _roll_min_max_variable(ndarray[numeric] values, return output -cdef _roll_min_max_fixed(ndarray[numeric] values, - ndarray[int64_t] starti, - ndarray[int64_t] endi, +cdef _roll_min_max_fixed(numeric[:] values, int64_t minp, int64_t win, bint is_max):