From 04e1f705c9f25e01c4c891468c53c16d3117ff17 Mon Sep 17 00:00:00 2001 From: ihsan Date: Wed, 31 Jul 2019 22:54:45 +0300 Subject: [PATCH 01/17] ENH: Implement weighted rolling var and std --- pandas/_libs/window.pyx | 129 ++++++++++ pandas/core/window.py | 354 +++++++++++++++------------- pandas/tests/window/test_moments.py | 20 +- pandas/tests/window/test_window.py | 2 +- 4 files changed, 343 insertions(+), 162 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 8de593ce36c86..1a354f14077ad 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1754,6 +1754,135 @@ def _roll_weighted_sum_mean(float64_t[:] values, float64_t[:] weights, return output +# ---------------------------------------------------------------------- +# Rolling var for weighted window + + +cdef inline float64_t calc_weighted_var(float64_t t, + float64_t sum_w, + Py_ssize_t win, + int ddof, + float64_t nobs, + int64_t minp) nogil: + cdef: + float64_t result + + # Variance is unchanged if no observation is added or removed + if (nobs >= minp) and (nobs > ddof): + + # pathological case + if nobs == 1: + result = 0 + else: + result = t * win / ((win - ddof) * sum_w) + if result < 0: + result = 0 + else: + result = NaN + + return result + + +cdef inline void add_weighted_var(float64_t val, + float64_t w, + float64_t *t, + float64_t *sum_w, + float64_t *mean, + float64_t *nobs) nogil: + cdef: + float64_t temp, q, r + + if isnan(val): + return + + nobs[0] = nobs[0] + 1 + + q = val - mean[0] + temp = sum_w[0] + w + r = q * w / temp + + mean[0] = mean[0] + r + t[0] = t[0] + r * sum_w[0] * q + sum_w[0] = temp + + +cdef inline void remove_weighted_var(float64_t val, + float64_t w, + float64_t *t, + float64_t *sum_w, + float64_t *mean, + float64_t *nobs) nogil: + cdef: + float64_t temp, q, r + + if notnan(val): + nobs[0] = nobs[0] - 1 + + if nobs[0]: + q = val - mean[0] + temp = sum_w[0] - w + r = q * w / temp + + mean[0] = mean[0] - r + t[0] = t[0] - r * sum_w[0] * q + sum_w[0] = temp + + else: + t[0] = 0 + sum_w[0] = 0 + mean[0] = 0 + + +def roll_weighted_var(float64_t[:] values, float64_t[:] weights, + int64_t minp, int ddof): + """ + Calculates weighted rolling variance using West's online algorithm + + Paper: https://dl.acm.org/citation.cfm?id=359153 + """ + cdef: + float64_t t = 0, sum_w = 0, mean = 0, nobs = 0 + float64_t val, pre_val, w, pre_w + Py_ssize_t i, n, win_n + float64_t[:] output + + n = len(values) + win_n = len(weights) + output = np.empty(n, dtype=float) + + with nogil: + + for i in range(win_n): + add_weighted_var(values[i], weights[i], &t, + &sum_w, &mean, &nobs) + + output[i] = calc_weighted_var(t, sum_w, win_n, + ddof, nobs, minp) + + for i in range(win_n, n): + val = values[i] + pre_val = values[i - win_n] + + w = weights[i % win_n] + pre_w = weights[(i - win_n) % win_n] + + if notnan(val): + if pre_val == pre_val: + remove_weighted_var(pre_val, pre_w, &t, + &sum_w, &mean, &nobs) + + add_weighted_var(val, w, &t, &sum_w, &mean, &nobs) + + elif pre_val == pre_val: + remove_weighted_var(pre_val, pre_w, &t, + &sum_w, &mean, &nobs) + + output[i] = calc_weighted_var(t, sum_w, win_n, + ddof, nobs, minp) + + return output + + # ---------------------------------------------------------------------- # Exponentially weighted moving average diff --git a/pandas/core/window.py b/pandas/core/window.py index 2199daa743655..bfb37ea54cf3f 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -5,7 +5,7 @@ from collections import defaultdict from datetime import timedelta from textwrap import dedent -from typing import Callable, List, Optional, Set, Union +from typing import Callable, List, Optional, Set, Tuple, Union import warnings import numpy as np @@ -173,6 +173,17 @@ def __getattr__(self, attr): def _dir_additions(self): return self.obj._dir_additions() + def _get_kwargs(self, **kwargs) -> Tuple[dict, dict]: + """ + Separate kwargs for rolling and window functions. + + Returns + ------- + tuple of (roll_kwargs : dict, window_kwargs : dict) + kwargs for rolling and window function + """ + return kwargs, dict() + def _get_window(self, other=None, **kwargs) -> int: """ Returns window lenght @@ -405,6 +416,8 @@ def _apply( ------- y : type of input """ + roll_kwargs, window_kwargs = self._get_kwargs(**kwargs) + if center is None: center = self.center @@ -412,7 +425,7 @@ def _apply( check_minp = _use_window if window is None: - window = self._get_window(**kwargs) + window = self._get_window(**window_kwargs) blocks, obj = self._create_blocks() block_list = list(blocks) @@ -445,7 +458,9 @@ def _apply( "in libwindow.{func}".format(func=func) ) - func = self._get_roll_func(cfunc, check_minp, index_as_array, **kwargs) + func = self._get_roll_func( + cfunc, check_minp, index_as_array, **roll_kwargs + ) # calculation function if center: @@ -612,6 +627,126 @@ def aggregate(self, func, *args, **kwargs): """ ) + _shared_docs["var"] = dedent( + """ + Calculate unbiased %(name)s variance. + + Normalized by N-1 by default. This can be changed using the `ddof` + argument. + + Parameters + ---------- + ddof : int, default 1 + Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + *args, **kwargs + For NumPy compatibility. No additional arguments are used. + + Returns + ------- + Series or DataFrame + Returns the same object type as the caller of the %(name)s calculation. + + See Also + -------- + Series.%(name)s : Calling object with Series data. + DataFrame.%(name)s : Calling object with DataFrames. + Series.var : Equivalent method for Series. + DataFrame.var : Equivalent method for DataFrame. + numpy.var : Equivalent method for Numpy array. + + Notes + ----- + The default `ddof` of 1 used in :meth:`Series.var` is different than the + default `ddof` of 0 in :func:`numpy.var`. + + A minimum of 1 period is required for the rolling calculation. + + Examples + -------- + >>> s = pd.Series([5, 5, 6, 7, 5, 5, 5]) + >>> s.rolling(3).var() + 0 NaN + 1 NaN + 2 0.333333 + 3 1.000000 + 4 1.000000 + 5 1.333333 + 6 0.000000 + dtype: float64 + + >>> s.expanding(3).var() + 0 NaN + 1 NaN + 2 0.333333 + 3 0.916667 + 4 0.800000 + 5 0.700000 + 6 0.619048 + dtype: float64 + """ + ) + + _shared_docs["std"] = dedent( + """ + Calculate %(name)s standard deviation. + + Normalized by N-1 by default. This can be changed using the `ddof` + argument. + + Parameters + ---------- + ddof : int, default 1 + Delta Degrees of Freedom. The divisor used in calculations + is ``N - ddof``, where ``N`` represents the number of elements. + *args, **kwargs + For NumPy compatibility. No additional arguments are used. + + Returns + ------- + Series or DataFrame + Returns the same object type as the caller of the %(name)s calculation. + + See Also + -------- + Series.%(name)s : Calling object with Series data. + DataFrame.%(name)s : Calling object with DataFrames. + Series.std : Equivalent method for Series. + DataFrame.std : Equivalent method for DataFrame. + numpy.std : Equivalent method for Numpy array. + + Notes + ----- + The default `ddof` of 1 used in Series.std is different than the default + `ddof` of 0 in numpy.std. + + A minimum of one period is required for the rolling calculation. + + Examples + -------- + >>> s = pd.Series([5, 5, 6, 7, 5, 5, 5]) + >>> s.rolling(3).std() + 0 NaN + 1 NaN + 2 0.577350 + 3 1.000000 + 4 1.000000 + 5 1.154701 + 6 0.000000 + dtype: float64 + + >>> s.expanding(3).std() + 0 NaN + 1 NaN + 2 0.577350 + 3 0.957427 + 4 0.894427 + 5 0.836660 + 6 0.786796 + dtype: float64 + """ + ) + class Window(_Window): """ @@ -785,10 +920,52 @@ def validate(self): else: raise ValueError("Invalid window {0}".format(window)) + def _get_kwargs(self, **kwargs) -> Tuple[dict, dict]: + """ + Validate arguments for the window function, then separate + them from arguments of rolling function. + + Returns + ------- + tuple of (roll_kwargs : dict, window_kwargs : dict) + kwargs for rolling and window function + """ + # the below may pop from kwargs + def _validate_win_type(win_type, kwargs): + arg_map = { + "kaiser": ["beta"], + "gaussian": ["std"], + "general_gaussian": ["power", "width"], + "slepian": ["width"], + "exponential": ["tau"], + } + + if win_type in arg_map: + win_args = _pop_args(win_type, arg_map[win_type], kwargs) + if win_type == "exponential": + # exponential window requires the first arg (center) + # to be set to None (necessary for symmetric window) + win_args.insert(0, None) + + return tuple([win_type] + win_args) + + return win_type + + def _pop_args(win_type, arg_names, kwargs): + msg = "%s window requires %%s" % win_type + all_args = [] + for n in arg_names: + if n not in kwargs: + raise ValueError(msg % n) + all_args.append(kwargs.pop(n)) + return all_args + + win_type = _validate_win_type(self.win_type, kwargs) + return kwargs, dict(window=win_type) + def _get_window(self, other=None, **kwargs) -> np.ndarray: """ - Provide validation for the window type, return the window - which has already been validated. + Get the window, weights. Parameters ---------- @@ -807,46 +984,15 @@ def _get_window(self, other=None, **kwargs) -> np.ndarray: elif is_integer(window): import scipy.signal as sig - # the below may pop from kwargs - def _validate_win_type(win_type, kwargs): - arg_map = { - "kaiser": ["beta"], - "gaussian": ["std"], - "general_gaussian": ["power", "width"], - "slepian": ["width"], - "exponential": ["tau"], - } - - if win_type in arg_map: - win_args = _pop_args(win_type, arg_map[win_type], kwargs) - if win_type == "exponential": - # exponential window requires the first arg (center) - # to be set to None (necessary for symmetric window) - win_args.insert(0, None) - - return tuple([win_type] + win_args) - - return win_type - - def _pop_args(win_type, arg_names, kwargs): - msg = "%s window requires %%s" % win_type - all_args = [] - for n in arg_names: - if n not in kwargs: - raise ValueError(msg % n) - all_args.append(kwargs.pop(n)) - return all_args - - win_type = _validate_win_type(self.win_type, kwargs) # GH #15662. `False` makes symmetric window, rather than periodic. - return sig.get_window(win_type, window, False).astype(float) + return sig.get_window(Nx=window, fftbins=False, **kwargs).astype(float) def _get_roll_func( self, cfunc: Callable, check_minp: Callable, index: np.ndarray, **kwargs ) -> Callable: def func(arg, window, min_periods=None, closed=None): minp = check_minp(min_periods, len(window)) - return cfunc(arg, window, minp) + return cfunc(arg, window, minp, **kwargs) return func @@ -924,6 +1070,18 @@ def mean(self, *args, **kwargs): nv.validate_window_func("mean", args, kwargs) return self._apply("roll_weighted_mean", **kwargs) + @Substitution(name="window") + @Appender(_shared_docs["var"]) + def var(self, ddof=1, *args, **kwargs): + nv.validate_window_func("var", args, kwargs) + return self._apply("roll_weighted_var", ddof=ddof, **kwargs) + + @Substitution(name="window") + @Appender(_shared_docs["std"]) + def std(self, ddof=1, *args, **kwargs): + nv.validate_window_func("std", args, kwargs) + return _zsqrt(self.var(ddof=ddof, **kwargs)) + class _GroupByMixin(GroupByMixin): """ @@ -1216,66 +1374,6 @@ def mean(self, *args, **kwargs): def median(self, **kwargs): return self._apply("roll_median_c", "median", **kwargs) - _shared_docs["std"] = dedent( - """ - Calculate %(name)s standard deviation. - - Normalized by N-1 by default. This can be changed using the `ddof` - argument. - - Parameters - ---------- - ddof : int, default 1 - Delta Degrees of Freedom. The divisor used in calculations - is ``N - ddof``, where ``N`` represents the number of elements. - *args, **kwargs - For NumPy compatibility. No additional arguments are used. - - Returns - ------- - Series or DataFrame - Returns the same object type as the caller of the %(name)s calculation. - - See Also - -------- - Series.%(name)s : Calling object with Series data. - DataFrame.%(name)s : Calling object with DataFrames. - Series.std : Equivalent method for Series. - DataFrame.std : Equivalent method for DataFrame. - numpy.std : Equivalent method for Numpy array. - - Notes - ----- - The default `ddof` of 1 used in Series.std is different than the default - `ddof` of 0 in numpy.std. - - A minimum of one period is required for the rolling calculation. - - Examples - -------- - >>> s = pd.Series([5, 5, 6, 7, 5, 5, 5]) - >>> s.rolling(3).std() - 0 NaN - 1 NaN - 2 0.577350 - 3 1.000000 - 4 1.000000 - 5 1.154701 - 6 0.000000 - dtype: float64 - - >>> s.expanding(3).std() - 0 NaN - 1 NaN - 2 0.577350 - 3 0.957427 - 4 0.894427 - 5 0.836660 - 6 0.786796 - dtype: float64 - """ - ) - def std(self, ddof=1, *args, **kwargs): nv.validate_window_func("std", args, kwargs) window = self._get_window() @@ -1291,66 +1389,6 @@ def f(arg, *args, **kwargs): f, "std", check_minp=_require_min_periods(1), ddof=ddof, **kwargs ) - _shared_docs["var"] = dedent( - """ - Calculate unbiased %(name)s variance. - - Normalized by N-1 by default. This can be changed using the `ddof` - argument. - - Parameters - ---------- - ddof : int, default 1 - Delta Degrees of Freedom. The divisor used in calculations - is ``N - ddof``, where ``N`` represents the number of elements. - *args, **kwargs - For NumPy compatibility. No additional arguments are used. - - Returns - ------- - Series or DataFrame - Returns the same object type as the caller of the %(name)s calculation. - - See Also - -------- - Series.%(name)s : Calling object with Series data. - DataFrame.%(name)s : Calling object with DataFrames. - Series.var : Equivalent method for Series. - DataFrame.var : Equivalent method for DataFrame. - numpy.var : Equivalent method for Numpy array. - - Notes - ----- - The default `ddof` of 1 used in :meth:`Series.var` is different than the - default `ddof` of 0 in :func:`numpy.var`. - - A minimum of 1 period is required for the rolling calculation. - - Examples - -------- - >>> s = pd.Series([5, 5, 6, 7, 5, 5, 5]) - >>> s.rolling(3).var() - 0 NaN - 1 NaN - 2 0.333333 - 3 1.000000 - 4 1.000000 - 5 1.333333 - 6 0.000000 - dtype: float64 - - >>> s.expanding(3).var() - 0 NaN - 1 NaN - 2 0.333333 - 3 0.916667 - 4 0.800000 - 5 0.700000 - 6 0.619048 - dtype: float64 - """ - ) - def var(self, ddof=1, *args, **kwargs): nv.validate_window_func("var", args, kwargs) return self._apply( diff --git a/pandas/tests/window/test_moments.py b/pandas/tests/window/test_moments.py index d860859958254..b42464a71981b 100644 --- a/pandas/tests/window/test_moments.py +++ b/pandas/tests/window/test_moments.py @@ -155,9 +155,23 @@ def test_cmov_window_frame(self): rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).mean() tm.assert_frame_equal(DataFrame(xp), rs) - # invalid method - with pytest.raises(AttributeError): - (DataFrame(vals).rolling(5, win_type="boxcar", center=True).std()) + # std + xp = np.array( + [ + [np.nan, np.nan], + [np.nan, np.nan], + [3.789706, 4.068313], + [3.429232, 3.237411], + [3.589269, 3.220810], + [3.405195, 2.380655], + [3.281839, 2.369869], + [3.676846, 1.801799], + [np.nan, np.nan], + [np.nan, np.nan], + ] + ) + rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).std() + tm.assert_frame_equal(DataFrame(xp), rs) # sum xp = np.array( diff --git a/pandas/tests/window/test_window.py b/pandas/tests/window/test_window.py index a6a56c98a9377..d2237f25cce6c 100644 --- a/pandas/tests/window/test_window.py +++ b/pandas/tests/window/test_window.py @@ -60,7 +60,7 @@ def test_numpy_compat(self, method): getattr(w, method)(dtype=np.float64) @td.skip_if_no_scipy - @pytest.mark.parametrize("arg", ["median", "var", "std", "kurt", "skew"]) + @pytest.mark.parametrize("arg", ["median", "kurt", "skew"]) def test_agg_function_support(self, arg): df = pd.DataFrame({"A": np.arange(5)}) roll = df.rolling(2, win_type="triang") From f62b06eaab6988a19ea0f59711ea41cab9e3aaab Mon Sep 17 00:00:00 2001 From: ihsan Date: Wed, 31 Jul 2019 23:05:37 +0300 Subject: [PATCH 02/17] Add whatsnew --- doc/source/whatsnew/v0.25.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.25.1.rst b/doc/source/whatsnew/v0.25.1.rst index fb67decb46b64..c6a8c5e67ea3b 100644 --- a/doc/source/whatsnew/v0.25.1.rst +++ b/doc/source/whatsnew/v0.25.1.rst @@ -16,7 +16,7 @@ Enhancements Other enhancements ^^^^^^^^^^^^^^^^^^ -- +- Implemented :meth:`pandas.core.window.Window.var` and :meth:`pandas.core.window.Window.std` functions (:issue:`26597`) - - From ea0b2db74b6f8dae1407bf6f8436f0a78581b9c6 Mon Sep 17 00:00:00 2001 From: ihsan Date: Thu, 1 Aug 2019 00:09:45 +0300 Subject: [PATCH 03/17] Remove whitespace --- pandas/_libs/window.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 1a354f14077ad..c8dd90845c64e 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1837,7 +1837,7 @@ def roll_weighted_var(float64_t[:] values, float64_t[:] weights, int64_t minp, int ddof): """ Calculates weighted rolling variance using West's online algorithm - + Paper: https://dl.acm.org/citation.cfm?id=359153 """ cdef: From 338754e7971f263d3e747259f0470ad36b897be9 Mon Sep 17 00:00:00 2001 From: ihsan Date: Thu, 1 Aug 2019 09:04:25 +0300 Subject: [PATCH 04/17] Pass updated kwargs to scipy's get_window --- pandas/core/window.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/window.py b/pandas/core/window.py index bfb37ea54cf3f..551ac1a229cfa 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -985,7 +985,8 @@ def _get_window(self, other=None, **kwargs) -> np.ndarray: import scipy.signal as sig # GH #15662. `False` makes symmetric window, rather than periodic. - return sig.get_window(Nx=window, fftbins=False, **kwargs).astype(float) + kwargs.update(dict(Nx=window, fftbins=False)) + return sig.get_window(**kwargs).astype(float) def _get_roll_func( self, cfunc: Callable, check_minp: Callable, index: np.ndarray, **kwargs From ff1aa8fe958033ff47056c57104c482f8e3e9425 Mon Sep 17 00:00:00 2001 From: ihsan Date: Thu, 1 Aug 2019 21:41:01 +0300 Subject: [PATCH 05/17] Make type changes --- pandas/_libs/window.pyx | 6 +++--- pandas/core/window.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index c8dd90845c64e..446eec08558c3 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1761,7 +1761,7 @@ def _roll_weighted_sum_mean(float64_t[:] values, float64_t[:] weights, cdef inline float64_t calc_weighted_var(float64_t t, float64_t sum_w, Py_ssize_t win, - int ddof, + unsigned int ddof, float64_t nobs, int64_t minp) nogil: cdef: @@ -1774,7 +1774,7 @@ cdef inline float64_t calc_weighted_var(float64_t t, if nobs == 1: result = 0 else: - result = t * win / ((win - ddof) * sum_w) + result = t * win / ((win - ddof) * sum_w) if result < 0: result = 0 else: @@ -1834,7 +1834,7 @@ cdef inline void remove_weighted_var(float64_t val, def roll_weighted_var(float64_t[:] values, float64_t[:] weights, - int64_t minp, int ddof): + int64_t minp, unsigned int ddof): """ Calculates weighted rolling variance using West's online algorithm diff --git a/pandas/core/window.py b/pandas/core/window.py index 551ac1a229cfa..5f2c84b0c1bf2 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -5,7 +5,7 @@ from collections import defaultdict from datetime import timedelta from textwrap import dedent -from typing import Callable, List, Optional, Set, Tuple, Union +from typing import Callable, Dict, List, Optional, Set, Tuple, Union import warnings import numpy as np @@ -173,7 +173,7 @@ def __getattr__(self, attr): def _dir_additions(self): return self.obj._dir_additions() - def _get_kwargs(self, **kwargs) -> Tuple[dict, dict]: + def _get_kwargs(self, **kwargs) -> Tuple[Dict, Dict]: """ Separate kwargs for rolling and window functions. @@ -920,7 +920,7 @@ def validate(self): else: raise ValueError("Invalid window {0}".format(window)) - def _get_kwargs(self, **kwargs) -> Tuple[dict, dict]: + def _get_kwargs(self, **kwargs) -> Tuple[Dict, Dict]: """ Validate arguments for the window function, then separate them from arguments of rolling function. From e58548e9407d3757fb7f4dee82f6d769cbab147e Mon Sep 17 00:00:00 2001 From: ihsan Date: Thu, 1 Aug 2019 21:42:39 +0300 Subject: [PATCH 06/17] Add test for var --- pandas/tests/window/test_moments.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pandas/tests/window/test_moments.py b/pandas/tests/window/test_moments.py index b42464a71981b..095d135daead9 100644 --- a/pandas/tests/window/test_moments.py +++ b/pandas/tests/window/test_moments.py @@ -173,6 +173,24 @@ def test_cmov_window_frame(self): rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).std() tm.assert_frame_equal(DataFrame(xp), rs) + # var + xp = np.array( + [ + [np.nan, np.nan], + [np.nan, np.nan], + [14.36187, 16.55117], + [11.75963, 10.48083], + [12.88285, 10.37362], + [11.59535, 5.66752], + [10.77047, 5.61628], + [13.51920, 3.24648], + [np.nan, np.nan], + [np.nan, np.nan], + ] + ) + rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).var() + tm.assert_frame_equal(DataFrame(xp), rs) + # sum xp = np.array( [ From 1273cf93e1da19370b1dc5806c3772ea3f07cf90 Mon Sep 17 00:00:00 2001 From: ihsan Date: Thu, 1 Aug 2019 21:48:49 +0300 Subject: [PATCH 07/17] Add docs --- pandas/_libs/window.pyx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 446eec08558c3..041d215570c51 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1789,6 +1789,12 @@ cdef inline void add_weighted_var(float64_t val, float64_t *sum_w, float64_t *mean, float64_t *nobs) nogil: + """ + Update mean (mean), sum of weights (sum_w) and sum of weighted + squared differences (t) to include value (val) and weight (w) + pair in variance calculation. + """ + cdef: float64_t temp, q, r @@ -1812,6 +1818,12 @@ cdef inline void remove_weighted_var(float64_t val, float64_t *sum_w, float64_t *mean, float64_t *nobs) nogil: + """ + Update mean (mean), sum of weights (sum_w) and sum of weighted + squared differences (t) to remove value (val) and weight (w) + pair from variance calculation. + """ + cdef: float64_t temp, q, r @@ -1836,10 +1848,11 @@ cdef inline void remove_weighted_var(float64_t val, def roll_weighted_var(float64_t[:] values, float64_t[:] weights, int64_t minp, unsigned int ddof): """ - Calculates weighted rolling variance using West's online algorithm + Calculates weighted rolling variance using West's online algorithm. Paper: https://dl.acm.org/citation.cfm?id=359153 """ + cdef: float64_t t = 0, sum_w = 0, mean = 0, nobs = 0 float64_t val, pre_val, w, pre_w From f13f0aacea63184dfb2331658c1f3a3f978ef66f Mon Sep 17 00:00:00 2001 From: ihsan Date: Thu, 1 Aug 2019 22:34:31 +0300 Subject: [PATCH 08/17] Add more typing --- pandas/core/window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/window.py b/pandas/core/window.py index 5f2c84b0c1bf2..70594e4829c44 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -920,7 +920,7 @@ def validate(self): else: raise ValueError("Invalid window {0}".format(window)) - def _get_kwargs(self, **kwargs) -> Tuple[Dict, Dict]: + def _get_kwargs(self, **kwargs) -> Tuple[Dict, Dict[str, Union[str, Tuple]]]: """ Validate arguments for the window function, then separate them from arguments of rolling function. From d25561cb6eace8042c9233d2fc883cf34481b2bc Mon Sep 17 00:00:00 2001 From: ihsan Date: Fri, 2 Aug 2019 08:54:54 +0300 Subject: [PATCH 09/17] Update docs --- pandas/_libs/window.pyx | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 041d215570c51..5175ab77f5881 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1760,7 +1760,7 @@ def _roll_weighted_sum_mean(float64_t[:] values, float64_t[:] weights, cdef inline float64_t calc_weighted_var(float64_t t, float64_t sum_w, - Py_ssize_t win, + Py_ssize_t win_n, unsigned int ddof, float64_t nobs, int64_t minp) nogil: @@ -1774,7 +1774,7 @@ cdef inline float64_t calc_weighted_var(float64_t t, if nobs == 1: result = 0 else: - result = t * win / ((win - ddof) * sum_w) + result = t * win_n / ((win_n - ddof) * sum_w) if result < 0: result = 0 else: @@ -1790,9 +1790,11 @@ cdef inline void add_weighted_var(float64_t val, float64_t *mean, float64_t *nobs) nogil: """ - Update mean (mean), sum of weights (sum_w) and sum of weighted - squared differences (t) to include value (val) and weight (w) - pair in variance calculation. + Update weighted mean (mean), sum of weights (sum_w) and sum of + weighted squared differences (t) to include value (val) and + weight (w) pair in variance calculation using West's method. + + Paper: https://dl.acm.org/citation.cfm?id=359153 """ cdef: @@ -1819,9 +1821,11 @@ cdef inline void remove_weighted_var(float64_t val, float64_t *mean, float64_t *nobs) nogil: """ - Update mean (mean), sum of weights (sum_w) and sum of weighted - squared differences (t) to remove value (val) and weight (w) - pair from variance calculation. + Update weighted mean (mean), sum of weights (sum_w) and sum + of weighted squared differences (t) to remove value (val) and + weight (w) pair from variance calculation using West's method. + + Paper: https://dl.acm.org/citation.cfm?id=359153 """ cdef: From 7840b294d930701e66f5715eaf8dd785e25fdc03 Mon Sep 17 00:00:00 2001 From: ihsan Date: Fri, 2 Aug 2019 19:53:39 +0300 Subject: [PATCH 10/17] Add docs --- doc/source/whatsnew/v0.25.1.rst | 2 +- doc/source/whatsnew/v1.0.0.rst | 2 +- pandas/_libs/window.pyx | 86 ++++++++++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/doc/source/whatsnew/v0.25.1.rst b/doc/source/whatsnew/v0.25.1.rst index 23a9067ea4c36..c80195af413f7 100644 --- a/doc/source/whatsnew/v0.25.1.rst +++ b/doc/source/whatsnew/v0.25.1.rst @@ -12,7 +12,7 @@ Enhancements Other enhancements ^^^^^^^^^^^^^^^^^^ -- Implemented :meth:`pandas.core.window.Window.var` and :meth:`pandas.core.window.Window.std` functions (:issue:`26597`) +- - - diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 08ae69312d4fd..ccd3e38e3f84c 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -31,7 +31,7 @@ Other enhancements .. _whatsnew_1000.api_breaking: -- +- Implemented :meth:`pandas.core.window.Window.var` and :meth:`pandas.core.window.Window.std` functions (:issue:`26597`) - Backwards incompatible API changes diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 5175ab77f5881..f7e1ff584c56d 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1764,6 +1764,32 @@ cdef inline float64_t calc_weighted_var(float64_t t, unsigned int ddof, float64_t nobs, int64_t minp) nogil: + """ + Calculate weighted variance for a window using West's method. + + Paper: https://dl.acm.org/citation.cfm?id=359153 + + Parameters + ---------- + t: float64_t + sum of weighted squared differences + sum_w: float64_t + sum of weights + win_n: Py_ssize_t + window size + ddof: unsigned int + delta degrees of freedom + nobs: float64_t + number of observations + minp: int64_t + minimum number of observations + + Returns + ------- + result : float64_t + weighted variance of the window + """ + cdef: float64_t result @@ -1790,11 +1816,26 @@ cdef inline void add_weighted_var(float64_t val, float64_t *mean, float64_t *nobs) nogil: """ - Update weighted mean (mean), sum of weights (sum_w) and sum of - weighted squared differences (t) to include value (val) and - weight (w) pair in variance calculation using West's method. + Update weighted mean, sum of weights and sum of weighted squared + differences to include value and weight pair in weighted variance + calculation using West's method. Paper: https://dl.acm.org/citation.cfm?id=359153 + + Parameters + ---------- + val: float64_t + window values + w: float64_t + window weights + t: float64_t + sum of weighted squared differences + sum_w: float64_t + sum of weights + mean: float64_t + weighted mean + nobs: float64_t + number of observations """ cdef: @@ -1821,11 +1862,26 @@ cdef inline void remove_weighted_var(float64_t val, float64_t *mean, float64_t *nobs) nogil: """ - Update weighted mean (mean), sum of weights (sum_w) and sum - of weighted squared differences (t) to remove value (val) and - weight (w) pair from variance calculation using West's method. + Update weighted mean, sum of weights and sum of weighted squared + differences to remove value and weight pair from weighted variance + calculation using West's method. Paper: https://dl.acm.org/citation.cfm?id=359153 + + Parameters + ---------- + val: float64_t + window values + w: float64_t + window weights + t: float64_t + sum of weighted squared differences + sum_w: float64_t + sum of weights + mean: float64_t + weighted mean + nobs: float64_t + number of observations """ cdef: @@ -1855,6 +1911,24 @@ def roll_weighted_var(float64_t[:] values, float64_t[:] weights, Calculates weighted rolling variance using West's online algorithm. Paper: https://dl.acm.org/citation.cfm?id=359153 + + Parameters + ---------- + values: float64_t[:] + values to roll window over + weights: float64_t[:] + array of weights whose lenght is window size + minp: int64_t + minimum number of observations to calculate + variance of a window + ddof: unsigned int + the divisor used in variance calculations + is the window size - ddof + + Returns + ------- + output: float64_t[:] + weighted variances of windows """ cdef: From 6922e4d482dc6eed90f0e500c18bc0f3019767b4 Mon Sep 17 00:00:00 2001 From: ihsan Date: Wed, 7 Aug 2019 08:54:36 +0300 Subject: [PATCH 11/17] Simplify extracting window args --- pandas/core/window.py | 63 +++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/pandas/core/window.py b/pandas/core/window.py index e011b3a7086a4..3bfe57ab9908e 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -173,24 +173,30 @@ def __getattr__(self, attr): def _dir_additions(self): return self.obj._dir_additions() - def _get_kwargs(self, **kwargs) -> Tuple[Dict, Dict]: + def _get_win_type(self, kwargs: Dict): """ - Separate kwargs for rolling and window functions. + Exists for compatibility, overriden by subclass Window. + + Parameters + ---------- + kwargs : dict + ignored, exists for compatibility Returns ------- - tuple of (roll_kwargs : dict, window_kwargs : dict) - kwargs for rolling and window function + None """ - return kwargs, dict() + return None - def _get_window(self, other=None, **kwargs) -> int: + def _get_window(self, other=None, win_type: Optional[str] = None) -> int: """ - Returns window length + Return window length. Parameters ---------- - other: + other : + ignored, exists for compatibility + win_type : ignored, exists for compatibility Returns @@ -416,7 +422,6 @@ def _apply( ------- y : type of input """ - roll_kwargs, window_kwargs = self._get_kwargs(**kwargs) if center is None: center = self.center @@ -425,7 +430,8 @@ def _apply( check_minp = _use_window if window is None: - window = self._get_window(**window_kwargs) + win_type = self._get_win_type(kwargs) + window = self._get_window(win_type=win_type) blocks, obj = self._create_blocks() block_list = list(blocks) @@ -458,9 +464,7 @@ def _apply( "in libwindow.{func}".format(func=func) ) - func = self._get_roll_func( - cfunc, check_minp, index_as_array, **roll_kwargs - ) + func = self._get_roll_func(cfunc, check_minp, index_as_array, **kwargs) # calculation function if center: @@ -920,15 +924,18 @@ def validate(self): else: raise ValueError("Invalid window {0}".format(window)) - def _get_kwargs(self, **kwargs) -> Tuple[Dict, Dict[str, Union[str, Tuple]]]: + def _get_win_type(self, kwargs: Dict) -> Union[str, Tuple]: """ - Validate arguments for the window function, then separate - them from arguments of rolling function. + Extract arguments for the window type, provide validation for it + and return the validated window type. + + Parameters + ---------- + kwargs : dict Returns ------- - tuple of (roll_kwargs : dict, window_kwargs : dict) - kwargs for rolling and window function + win_type : str, or tuple """ # the below may pop from kwargs def _validate_win_type(win_type, kwargs): @@ -960,17 +967,20 @@ def _pop_args(win_type, arg_names, kwargs): all_args.append(kwargs.pop(n)) return all_args - win_type = _validate_win_type(self.win_type, kwargs) - return kwargs, dict(window=win_type) + return _validate_win_type(self.win_type, kwargs) - def _get_window(self, other=None, **kwargs) -> np.ndarray: + def _get_window( + self, other=None, win_type: Optional[Union[str, Tuple]] = None + ) -> np.ndarray: """ Get the window, weights. Parameters ---------- - other: + other : ignored, exists for compatibility + win_type : str, or tuple + type of window to create Returns ------- @@ -985,8 +995,7 @@ def _get_window(self, other=None, **kwargs) -> np.ndarray: import scipy.signal as sig # GH #15662. `False` makes symmetric window, rather than periodic. - kwargs.update(dict(Nx=window, fftbins=False)) - return sig.get_window(**kwargs).astype(float) + return sig.get_window(win_type, window, False).astype(float) def _get_roll_func( self, cfunc: Callable, check_minp: Callable, index: np.ndarray, **kwargs @@ -2069,15 +2078,17 @@ def __init__(self, obj, min_periods=1, center=False, axis=0, **kwargs): def _constructor(self): return Expanding - def _get_window(self, other=None, **kwargs): + def _get_window(self, other=None, win_type=None): """ Get the window length over which to perform some operation. Parameters ---------- - other : object, default None + other : Series, DataFrame, or ndarray, optional The other object that is involved in the operation. Such an object is involved for operations like covariance. + win_type : + ignored, exists for compatibility Returns ------- From 6f0f1a583d591291408cab7ce9cc8dfd8636b5c0 Mon Sep 17 00:00:00 2001 From: ihsansecer Date: Mon, 9 Sep 2019 22:23:58 +0300 Subject: [PATCH 12/17] Add version to docstring, add reference --- doc/source/reference/window.rst | 2 ++ pandas/core/window/expanding.py | 4 ++-- pandas/core/window/rolling.py | 10 ++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/source/reference/window.rst b/doc/source/reference/window.rst index 2f6addf607877..d09ac0d1fa7f7 100644 --- a/doc/source/reference/window.rst +++ b/doc/source/reference/window.rst @@ -34,6 +34,8 @@ Standard moving window functions Rolling.quantile Window.mean Window.sum + Window.var + Window.std .. _api.functions_expanding: diff --git a/pandas/core/window/expanding.py b/pandas/core/window/expanding.py index 47bd8f2ec593b..c3b7531ce5904 100644 --- a/pandas/core/window/expanding.py +++ b/pandas/core/window/expanding.py @@ -181,13 +181,13 @@ def mean(self, *args, **kwargs): def median(self, **kwargs): return super().median(**kwargs) - @Substitution(name="expanding") + @Substitution(name="expanding", versionadded="") @Appender(_shared_docs["std"]) def std(self, ddof=1, *args, **kwargs): nv.validate_expanding_func("std", args, kwargs) return super().std(ddof=ddof, **kwargs) - @Substitution(name="expanding") + @Substitution(name="expanding", versionadded="") @Appender(_shared_docs["var"]) def var(self, ddof=1, *args, **kwargs): nv.validate_expanding_func("var", args, kwargs) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index 1b53c9fa3692e..bad2408653639 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -635,6 +635,8 @@ def aggregate(self, func, *args, **kwargs): """ Calculate unbiased %(name)s variance. + %(versionadded)s + Normalized by N-1 by default. This can be changed using the `ddof` argument. @@ -1080,13 +1082,13 @@ def mean(self, *args, **kwargs): nv.validate_window_func("mean", args, kwargs) return self._apply("roll_weighted_mean", **kwargs) - @Substitution(name="window") + @Substitution(name="window", versionadded=".. versionadded:: 1.0.0") @Appender(_shared_docs["var"]) def var(self, ddof=1, *args, **kwargs): nv.validate_window_func("var", args, kwargs) return self._apply("roll_weighted_var", ddof=ddof, **kwargs) - @Substitution(name="window") + @Substitution(name="window", versionadded=".. versionadded:: 1.0.0") @Appender(_shared_docs["std"]) def std(self, ddof=1, *args, **kwargs): nv.validate_window_func("std", args, kwargs) @@ -1889,13 +1891,13 @@ def mean(self, *args, **kwargs): def median(self, **kwargs): return super().median(**kwargs) - @Substitution(name="rolling") + @Substitution(name="rolling", versionadded="") @Appender(_shared_docs["std"]) def std(self, ddof=1, *args, **kwargs): nv.validate_rolling_func("std", args, kwargs) return super().std(ddof=ddof, **kwargs) - @Substitution(name="rolling") + @Substitution(name="rolling", versionadded="") @Appender(_shared_docs["var"]) def var(self, ddof=1, *args, **kwargs): nv.validate_rolling_func("var", args, kwargs) From b54dea00a51e53b4bf0c5f58f3c9115e4ba9cbc4 Mon Sep 17 00:00:00 2001 From: ihsansecer Date: Tue, 10 Sep 2019 19:38:46 +0300 Subject: [PATCH 13/17] Fix multiline issue --- pandas/core/window/rolling.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index bad2408653639..e44076549f0ca 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -636,7 +636,6 @@ def aggregate(self, func, *args, **kwargs): Calculate unbiased %(name)s variance. %(versionadded)s - Normalized by N-1 by default. This can be changed using the `ddof` argument. @@ -697,6 +696,7 @@ def aggregate(self, func, *args, **kwargs): """ Calculate %(name)s standard deviation. + %(versionadded)s Normalized by N-1 by default. This can be changed using the `ddof` argument. @@ -1082,13 +1082,13 @@ def mean(self, *args, **kwargs): nv.validate_window_func("mean", args, kwargs) return self._apply("roll_weighted_mean", **kwargs) - @Substitution(name="window", versionadded=".. versionadded:: 1.0.0") + @Substitution(name="window", versionadded=".. versionadded:: 1.0.0\n") @Appender(_shared_docs["var"]) def var(self, ddof=1, *args, **kwargs): nv.validate_window_func("var", args, kwargs) return self._apply("roll_weighted_var", ddof=ddof, **kwargs) - @Substitution(name="window", versionadded=".. versionadded:: 1.0.0") + @Substitution(name="window", versionadded=".. versionadded:: 1.0.0\n") @Appender(_shared_docs["std"]) def std(self, ddof=1, *args, **kwargs): nv.validate_window_func("std", args, kwargs) From 4bb954f5f37b50874043c090928de5e94cd0c18a Mon Sep 17 00:00:00 2001 From: ihsansecer Date: Tue, 10 Sep 2019 22:11:25 +0300 Subject: [PATCH 14/17] Fix multiline issue --- pandas/core/window/rolling.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py index e44076549f0ca..ba3d26fa6841e 100644 --- a/pandas/core/window/rolling.py +++ b/pandas/core/window/rolling.py @@ -634,7 +634,6 @@ def aggregate(self, func, *args, **kwargs): _shared_docs["var"] = dedent( """ Calculate unbiased %(name)s variance. - %(versionadded)s Normalized by N-1 by default. This can be changed using the `ddof` argument. @@ -695,7 +694,6 @@ def aggregate(self, func, *args, **kwargs): _shared_docs["std"] = dedent( """ Calculate %(name)s standard deviation. - %(versionadded)s Normalized by N-1 by default. This can be changed using the `ddof` argument. @@ -1082,13 +1080,13 @@ def mean(self, *args, **kwargs): nv.validate_window_func("mean", args, kwargs) return self._apply("roll_weighted_mean", **kwargs) - @Substitution(name="window", versionadded=".. versionadded:: 1.0.0\n") + @Substitution(name="window", versionadded="\n.. versionadded:: 1.0.0\n") @Appender(_shared_docs["var"]) def var(self, ddof=1, *args, **kwargs): nv.validate_window_func("var", args, kwargs) return self._apply("roll_weighted_var", ddof=ddof, **kwargs) - @Substitution(name="window", versionadded=".. versionadded:: 1.0.0\n") + @Substitution(name="window", versionadded="\n.. versionadded:: 1.0.0\n") @Appender(_shared_docs["std"]) def std(self, ddof=1, *args, **kwargs): nv.validate_window_func("std", args, kwargs) From 3fa6028e562db99fc11b12021000d7afccb4d9be Mon Sep 17 00:00:00 2001 From: ihsansecer Date: Tue, 8 Oct 2019 23:41:18 +0100 Subject: [PATCH 15/17] Remove incoming change line --- doc/source/whatsnew/v1.0.0.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 8bc7fa8012b21..41708d252fa2e 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -119,7 +119,6 @@ Pandas has added a `pyproject.toml `_ cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``. ->>>>>>> upstream/master .. _whatsnew_1000.api_breaking: From 221a5229705ecd92fdf6e05de89b477c8d3be024 Mon Sep 17 00:00:00 2001 From: ihsansecer Date: Tue, 8 Oct 2019 23:58:13 +0100 Subject: [PATCH 16/17] Parameterize window test --- pandas/tests/window/test_moments.py | 171 ++++++++++++++-------------- 1 file changed, 85 insertions(+), 86 deletions(-) diff --git a/pandas/tests/window/test_moments.py b/pandas/tests/window/test_moments.py index 6d9fe81b9c378..36a0ddb3e02d7 100644 --- a/pandas/tests/window/test_moments.py +++ b/pandas/tests/window/test_moments.py @@ -119,96 +119,95 @@ def test_cmov_window_corner(self): assert len(result) == 5 @td.skip_if_no_scipy - def test_cmov_window_frame(self): + @pytest.mark.parametrize( + "f,xp", + [ + ( + "mean", + [ + [np.nan, np.nan], + [np.nan, np.nan], + [9.252, 9.392], + [8.644, 9.906], + [8.87, 10.208], + [6.81, 8.588], + [7.792, 8.644], + [9.05, 7.824], + [np.nan, np.nan], + [np.nan, np.nan], + ], + ), + ( + "std", + [ + [np.nan, np.nan], + [np.nan, np.nan], + [3.789706, 4.068313], + [3.429232, 3.237411], + [3.589269, 3.220810], + [3.405195, 2.380655], + [3.281839, 2.369869], + [3.676846, 1.801799], + [np.nan, np.nan], + [np.nan, np.nan], + ], + ), + ( + "var", + [ + [np.nan, np.nan], + [np.nan, np.nan], + [14.36187, 16.55117], + [11.75963, 10.48083], + [12.88285, 10.37362], + [11.59535, 5.66752], + [10.77047, 5.61628], + [13.51920, 3.24648], + [np.nan, np.nan], + [np.nan, np.nan], + ], + ), + ( + "sum", + [ + [np.nan, np.nan], + [np.nan, np.nan], + [46.26, 46.96], + [43.22, 49.53], + [44.35, 51.04], + [34.05, 42.94], + [38.96, 43.22], + [45.25, 39.12], + [np.nan, np.nan], + [np.nan, np.nan], + ], + ), + ], + ) + def test_cmov_window_frame(self, f, xp): # Gh 8238 - vals = np.array( - [ - [12.18, 3.64], - [10.18, 9.16], - [13.24, 14.61], - [4.51, 8.11], - [6.15, 11.44], - [9.14, 6.21], - [11.31, 10.67], - [2.94, 6.51], - [9.42, 8.39], - [12.44, 7.34], - ] - ) - - xp = np.array( - [ - [np.nan, np.nan], - [np.nan, np.nan], - [9.252, 9.392], - [8.644, 9.906], - [8.87, 10.208], - [6.81, 8.588], - [7.792, 8.644], - [9.05, 7.824], - [np.nan, np.nan], - [np.nan, np.nan], - ] - ) - - # DataFrame - rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).mean() - tm.assert_frame_equal(DataFrame(xp), rs) - - # std - xp = np.array( - [ - [np.nan, np.nan], - [np.nan, np.nan], - [3.789706, 4.068313], - [3.429232, 3.237411], - [3.589269, 3.220810], - [3.405195, 2.380655], - [3.281839, 2.369869], - [3.676846, 1.801799], - [np.nan, np.nan], - [np.nan, np.nan], - ] - ) - rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).std() - tm.assert_frame_equal(DataFrame(xp), rs) - - # var - xp = np.array( - [ - [np.nan, np.nan], - [np.nan, np.nan], - [14.36187, 16.55117], - [11.75963, 10.48083], - [12.88285, 10.37362], - [11.59535, 5.66752], - [10.77047, 5.61628], - [13.51920, 3.24648], - [np.nan, np.nan], - [np.nan, np.nan], - ] + df = DataFrame( + np.array( + [ + [12.18, 3.64], + [10.18, 9.16], + [13.24, 14.61], + [4.51, 8.11], + [6.15, 11.44], + [9.14, 6.21], + [11.31, 10.67], + [2.94, 6.51], + [9.42, 8.39], + [12.44, 7.34], + ] + ) ) - rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).var() - tm.assert_frame_equal(DataFrame(xp), rs) + xp = DataFrame(np.array(xp)) - # sum - xp = np.array( - [ - [np.nan, np.nan], - [np.nan, np.nan], - [46.26, 46.96], - [43.22, 49.53], - [44.35, 51.04], - [34.05, 42.94], - [38.96, 43.22], - [45.25, 39.12], - [np.nan, np.nan], - [np.nan, np.nan], - ] - ) + roll = df.rolling(5, win_type="boxcar", center=True) + rs = getattr(roll, f)() - rs = DataFrame(vals).rolling(5, win_type="boxcar", center=True).sum() - tm.assert_frame_equal(DataFrame(xp), rs) + tm.assert_frame_equal(xp, rs) @td.skip_if_no_scipy def test_cmov_window_na_min_periods(self): From b40dc0a0d382121186dc240448ca3e95fbd8dc10 Mon Sep 17 00:00:00 2001 From: ihsansecer Date: Wed, 9 Oct 2019 00:00:30 +0100 Subject: [PATCH 17/17] Remove blank line --- doc/source/whatsnew/v1.0.0.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 41708d252fa2e..aa570203d0404 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -99,7 +99,6 @@ See :ref:`text.types` for more. Other enhancements ^^^^^^^^^^^^^^^^^^ - - :meth:`DataFrame.to_string` added the ``max_colwidth`` parameter to control when wide columns are truncated (:issue:`9784`) - :meth:`MultiIndex.from_product` infers level names from inputs if not explicitly provided (:issue:`27292`) - :meth:`DataFrame.to_latex` now accepts ``caption`` and ``label`` arguments (:issue:`25436`)