ENH: Rolling window endpoints inclusion

closes pandas-dev#13965

Author: carlosdanielcsantos <[email protected]>
Author: carlosdanielcsantos <[email protected]>
Author: carlosdanielcsantos <[email protected]>

Closes pandas-dev#15795 from carlosdanielcsantos/rwindow-endpoints-inclusion and squashes the following commits:

aad97dc [carlosdanielcsantos] Updating docs
568c12f [carlosdanielcsantos] Innocuous change to rerun tests
037b84e [carlosdanielcsantos] Fixing style
c18a31b [carlosdanielcsantos] Fixing test of assertion of closed parameter in fixed windows Style corrections
90dfb0c [carlosdanielcsantos] Correcting bug in window validation
8bd336a [carlosdanielcsantos] Almost there
306b9f7 [carlosdanielcsantos] Commiting progress on default=None. Still not tested Adding computation.rst section (still not written)
ec4bbc7 [carlosdanielcsantos] Changing l_closed and r_closed variable names
0e8e65c [carlosdanielcsantos] Adding doc-strings and PEP8 corrections
5eaf3b4 [carlosdanielcsantos] str closed -> object closed Adding test of assert for closed parameter Adding assert for closed parameter in get_window_indexer
2cf6804 [carlosdanielcsantos] Time-based windows working
da034bf [carlosdanielcsantos] Commiting progress
34f1309 [carlosdanielcsantos] Adding window slicing endpoint inclusion selection to VariableWindowIndexer

Author: carlosdanielcsantos

doc/source/computation.rst

@@ -459,6 +459,48 @@ default of the index) in a DataFrame.
    dft
    dft.rolling('2s', on='foo').sum()
 
+.. _stats.rolling_window.endpoints:
+
+Rolling Window Endpoints
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.20.0
+
+The inclusion of the interval endpoints in rolling window calculations can be specified with the ``closed``
+parameter:
+
+.. csv-table::
+    :header: "``closed``", "Description", "Default for"
+    :widths: 20, 30, 30
+
+    ``right``, close right endpoint, time-based windows
+    ``left``, close left endpoint,
+    ``both``, close both endpoints, fixed windows
+    ``neither``, open endpoints,
+
+For example, having the right endpoint open is useful in many problems that require that there is no contamination
+from present information back to past information. This allows the rolling window to compute statistics
+"up to that point in time", but not including that point in time.
+
+.. ipython:: python
+
+   df = pd.DataFrame({'x': 1},
+                     index = [pd.Timestamp('20130101 09:00:01'),
+                              pd.Timestamp('20130101 09:00:02'),
+                              pd.Timestamp('20130101 09:00:03'),
+                              pd.Timestamp('20130101 09:00:04'),
+                              pd.Timestamp('20130101 09:00:06')])
+
+   df["right"] = df.rolling('2s', closed='right').x.sum()  # default
+   df["both"] = df.rolling('2s', closed='both').x.sum()
+   df["left"] = df.rolling('2s', closed='left').x.sum()
+   df["neither"] = df.rolling('2s', closed='neither').x.sum()
+
+   df
+
+Currently, this feature is only implemented for time-based windows.
+For fixed windows, the closed parameter cannot be set and the rolling window will always have both endpoints closed.
+
 .. _stats.moments.ts-versus-resampling:
 
 Time-aware Rolling vs. Resampling

doc/source/whatsnew/v0.20.0.txt

@@ -319,6 +319,7 @@ To convert a ``SparseDataFrame`` back to sparse SciPy matrix in COO format, you
 Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 
+- ``DataFrame.rolling()`` now accepts the parameter ``closed='right'|'left'|'both'|'neither'`` to choose the rolling window endpoint closedness. See the :ref:`documentation <stats.rolling_window.endpoints>` (:issue:`13965`)
 - Integration with the ``feather-format``, including a new top-level ``pd.read_feather()`` and ``DataFrame.to_feather()`` method, see :ref:`here <io.feather>`.
 - ``Series.str.replace()`` now accepts a callable, as replacement, which is passed to ``re.sub`` (:issue:`15055`)
 - ``Series.str.replace()`` now accepts a compiled regular expression as a pattern (:issue:`15446`)

pandas/core/generic.py

@@ -5962,12 +5962,12 @@ def _add_series_or_dataframe_operations(cls):
 
         @Appender(rwindow.rolling.__doc__)
         def rolling(self, window, min_periods=None, freq=None, center=False,
-                    win_type=None, on=None, axis=0):
+                    win_type=None, on=None, axis=0, closed=None):
             axis = self._get_axis_number(axis)
             return rwindow.rolling(self, window=window,
                                    min_periods=min_periods, freq=freq,
                                    center=center, win_type=win_type,
-                                   on=on, axis=axis)
+                                   on=on, axis=axis, closed=closed)
 
         cls.rolling = rolling
 

pandas/core/window.py

@@ -56,11 +56,12 @@
 
 class _Window(PandasObject, SelectionMixin):
     _attributes = ['window', 'min_periods', 'freq', 'center', 'win_type',
-                   'axis', 'on']
+                   'axis', 'on', 'closed']
     exclusions = set()
 
     def __init__(self, obj, window=None, min_periods=None, freq=None,
-                 center=False, win_type=None, axis=0, on=None, **kwargs):
+                 center=False, win_type=None, axis=0, on=None, closed=None,
+                 **kwargs):
 
         if freq is not None:
             warnings.warn("The freq kw is deprecated and will be removed in a "
@@ -71,6 +72,7 @@ def __init__(self, obj, window=None, min_periods=None, freq=None,
         self.blocks = []
         self.obj = obj
         self.on = on
+        self.closed = closed
         self.window = window
         self.min_periods = min_periods
         self.freq = freq
@@ -101,6 +103,10 @@ def validate(self):
         if self.min_periods is not None and not \
            is_integer(self.min_periods):
             raise ValueError("min_periods must be an integer")
+        if self.closed is not None and self.closed not in \
+           ['right', 'both', 'left', 'neither']:
+            raise ValueError("closed must be 'right', 'left', 'both' or "
+                             "'neither'")
 
     def _convert_freq(self, how=None):
         """ resample according to the how, return a new object """
@@ -374,8 +380,14 @@ class Window(_Window):
     on : string, optional
         For a DataFrame, column on which to calculate
         the rolling window, rather than the index
+    closed : string, default None
+        Make the interval closed on the 'right', 'left', 'both' or
+        'neither' endpoints.
+        For offset-based windows, it defaults to 'right'.
+        For fixed windows, defaults to 'both'. Remaining cases not implemented
+        for fixed windows.
 
-        .. versionadded:: 0.19.0
+        .. versionadded:: 0.20.0
 
     axis : int or string, default 0
 
@@ -717,12 +729,12 @@ def _apply(self, func, name=None, window=None, center=None,
                     raise ValueError("we do not support this function "
                                      "in _window.{0}".format(func))
 
-                def func(arg, window, min_periods=None):
+                def func(arg, window, min_periods=None, closed=None):
                     minp = check_minp(min_periods, window)
                     # ensure we are only rolling on floats
                     arg = _ensure_float64(arg)
                     return cfunc(arg,
-                                 window, minp, indexi, **kwargs)
+                                 window, minp, indexi, closed, **kwargs)
 
             # calculation function
             if center:
@@ -731,11 +743,13 @@ def func(arg, window, min_periods=None):
 
                 def calc(x):
                     return func(np.concatenate((x, additional_nans)),
-                                window, min_periods=self.min_periods)
+                                window, min_periods=self.min_periods,
+                                closed=self.closed)
             else:
 
                 def calc(x):
-                    return func(x, window, min_periods=self.min_periods)
+                    return func(x, window, min_periods=self.min_periods,
+                                closed=self.closed)
 
             with np.errstate(all='ignore'):
                 if values.ndim > 1:
@@ -768,7 +782,8 @@ def count(self):
         for b in blocks:
             result = b.notnull().astype(int)
             result = self._constructor(result, window=window, min_periods=0,
-                                       center=self.center).sum()
+                                       center=self.center,
+                                       closed=self.closed).sum()
             results.append(result)
 
         return self._wrap_results(results, blocks, obj)
@@ -789,11 +804,10 @@ def apply(self, func, args=(), kwargs={}):
         offset = _offset(window, self.center)
         index, indexi = self._get_index()
 
-        def f(arg, window, min_periods):
+        def f(arg, window, min_periods, closed):
             minp = _use_window(min_periods, window)
-            return _window.roll_generic(arg, window, minp, indexi,
-                                        offset, func, args,
-                                        kwargs)
+            return _window.roll_generic(arg, window, minp, indexi, closed,
+                                        offset, func, args, kwargs)
 
         return self._apply(f, func, args=args, kwargs=kwargs,
                            center=False)
@@ -864,7 +878,7 @@ def std(self, ddof=1, *args, **kwargs):
         def f(arg, *args, **kwargs):
             minp = _require_min_periods(1)(self.min_periods, window)
             return _zsqrt(_window.roll_var(arg, window, minp, indexi,
-                                           ddof))
+                                           self.closed, ddof))
 
         return self._apply(f, 'std', check_minp=_require_min_periods(1),
                            ddof=ddof, **kwargs)
@@ -911,7 +925,7 @@ def quantile(self, quantile, **kwargs):
         def f(arg, *args, **kwargs):
             minp = _use_window(self.min_periods, window)
             return _window.roll_quantile(arg, window, minp, indexi,
-                                         quantile)
+                                         self.closed, quantile)
 
         return self._apply(f, 'quantile', quantile=quantile,
                            **kwargs)
@@ -1044,6 +1058,10 @@ def validate(self):
         elif self.window < 0:
             raise ValueError("window must be non-negative")
 
+        if not self.is_datetimelike and self.closed is not None:
+            raise ValueError("closed only implemented for datetimelike "
+                             "and offset based windows")
+
     def _validate_monotonic(self):
         """ validate on is monotonic """
         if not self._on.is_monotonic: