ENH: anchor resampling frequencies like 5minute that evenly subdivide one day in resampling to always get regular intervals. a bit more testing needed, but close #1165

Author: wesm

pandas/core/frame.py

@@ -32,7 +32,6 @@
 from pandas.core.internals import BlockManager, make_block, form_blocks
 from pandas.core.series import Series, _radd_compat
 from pandas.compat.scipy import scoreatpercentile as _quantile
-from pandas.tseries.index import DatetimeIndex
 from pandas.tseries.period import PeriodIndex
 from pandas.util import py3compat
 from pandas.util.terminal import get_terminal_size

pandas/core/internals.py

@@ -1013,7 +1013,8 @@ def form_blocks(data, axes):
         blocks.append(int_block)
 
     if len(datetime_dict):
-        datetime_block = _simple_blockify(datetime_dict, items, np.dtype('M8[us]'))
+        datetime_block = _simple_blockify(datetime_dict, items,
+                                          np.dtype('M8[us]'))
         blocks.append(datetime_block)
 
     if len(bool_dict):

pandas/tseries/index.py

@@ -1138,7 +1138,7 @@ def _generate_regular_range(start, end, periods, offset):
         raise ValueError('Must specify two of start, end, or periods')
 
     if isinstance(offset, Tick):
-        stride = offset.us_stride()
+        stride = offset.micros
         if periods is None:
             b = Timestamp(start).value
             e = Timestamp(end).value

pandas/tseries/offsets.py

@@ -962,7 +962,8 @@ def delta(self):
 
         return self._delta
 
-    def us_stride(self):
+    @property
+    def micros(self):
         return _delta_to_microseconds(self.delta)
 
     def apply(self, other):

pandas/tseries/resample.py

@@ -233,12 +233,22 @@ def _make_time_bins(axis, freq, begin=None, end=None,
     return binner, bins, labels
 
 def _get_range_edges(axis, begin, end, offset, closed='left'):
+    from pandas.tseries.offsets import Tick, _delta_to_microseconds
     if isinstance(offset, basestring):
         offset = to_offset(offset)
 
     if not isinstance(offset, DateOffset):
         raise ValueError("Rule not a recognized offset")
 
+    if isinstance(offset, Tick):
+        day_micros = _delta_to_microseconds(timedelta(1))
+        # #1165
+        if ((day_micros % offset.micros) == 0 and begin is None
+            and end is None):
+            return _adjust_dates_anchored(axis[0], axis[-1], offset,
+                                          closed=closed)
+
+
     if begin is None:
         if closed == 'left':
             first = Timestamp(offset.rollback(axis[0]))
@@ -255,6 +265,45 @@ def _get_range_edges(axis, begin, end, offset, closed='left'):
 
     return first, last
 
+
+def _adjust_dates_anchored(first, last, offset, closed='right'):
+    from pandas.tseries.tools import normalize_date
+
+    start_day_micros = Timestamp(normalize_date(first)).value
+    last_day_micros = Timestamp(normalize_date(last)).value
+
+    foffset = (first.value - start_day_micros) % offset.micros
+    loffset = (last.value - last_day_micros) % offset.micros
+
+    if closed == 'right':
+        if foffset > 0:
+            # roll back
+            fresult = first.value - foffset
+        else:
+            fresult = first.value - offset.micros
+
+        if loffset > 0:
+            # roll forward
+            lresult = last.value + (offset.micros - loffset)
+        else:
+            # already the end of the road
+            lresult = last.value
+    else:  # closed == 'left'
+        if foffset > 0:
+            fresult = first.value - foffset
+        else:
+            # start of the road
+            fresult = first.value
+
+        if loffset > 0:
+            # roll forward
+            lresult = last.value + (offset.micros - loffset)
+        else:
+            lresult = last.value + offset.micros
+
+    return Timestamp(fresult), Timestamp(lresult)
+
+
 def asfreq(obj, freq, method=None, how=None):
     """
     Utility frequency conversion method for Series/DataFrame

pandas/tseries/tests/test_resample.py

@@ -347,6 +347,31 @@ def test_resample_panel_numpy(self):
         expected = panel.resample('M', how='mean', axis=1)
         tm.assert_panel_equal(result, expected)
 
+    def test_resample_anchored_ticks(self):
+        # If a fixed delta (5 minute, 4 hour) evenly divides a day, we should
+        # "anchor" the origin at midnight so we get regular intervals rather
+        # than starting from the first timestamp which might start in the middle
+        # of a desired interval
+
+        rng = date_range('1/1/2000 04:00:00', periods=86400, freq='s')
+        ts = Series(np.random.randn(len(rng)), index=rng)
+        ts[:2] = np.nan # so results are the same
+
+        freqs = ['t', '5t', '15t', '30t', '4h', '12h']
+        for freq in freqs:
+            result = ts[2:].resample(freq, closed='left', label='left')
+            expected = ts.resample(freq, closed='left', label='left')
+            assert_series_equal(result, expected)
+
+    def test_resample_daily_anchored(self):
+        rng = date_range('1/1/2000 0:00:00', periods=10000, freq='T')
+        ts = Series(np.random.randn(len(rng)), index=rng)
+        ts[:2] = np.nan # so results are the same
+
+        result = ts[2:].resample('D', closed='left', label='left')
+        expected = ts.resample('D', closed='left', label='left')
+        assert_series_equal(result, expected)
+
 
 def _simple_ts(start, end, freq='D'):
     rng = date_range(start, end, freq=freq)

pandas/tseries/tests/test_util.py

@@ -6,17 +6,17 @@
 from pandas import Series, date_range
 import pandas.util.testing as tm
 
-from pandas.tseries.util import convert_to_annual, isleapyear
+from pandas.tseries.util import pivot_annual, isleapyear
 
-class TestConvertAnnual(unittest.TestCase):
+class TestPivotAnnual(unittest.TestCase):
     """
-    New pandas of scikits.timeseries convert_to_annual
+    New pandas of scikits.timeseries pivot_annual
     """
     def test_daily(self):
         rng = date_range('1/1/2000', '12/31/2004', freq='D')
         ts = Series(np.random.randn(len(rng)), index=rng)
 
-        annual = convert_to_annual(ts, 'D')
+        annual = pivot_annual(ts, 'D')
 
         doy = ts.index.dayofyear
         doy[(-isleapyear(ts.index.year)) & (doy >= 60)] += 1
@@ -40,7 +40,7 @@ def test_monthly(self):
         rng = date_range('1/1/2000', '12/31/2004', freq='M')
         ts = Series(np.random.randn(len(rng)), index=rng)
 
-        annual = convert_to_annual(ts, 'M')
+        annual = pivot_annual(ts, 'M')
 
         month = ts.index.month
 
@@ -49,13 +49,13 @@ def test_monthly(self):
             subset.index = [x.year for x in subset.index]
             tm.assert_series_equal(annual[i].dropna(), subset)
 
-    def test_interval_monthly(self):
+    def test_period_monthly(self):
         pass
 
-    def test_interval_daily(self):
+    def test_period_daily(self):
         pass
 
-    def test_interval_weekly(self):
+    def test_period_weekly(self):
         pass
 
 if __name__ == '__main__':

pandas/tseries/util.py

@@ -3,7 +3,7 @@
 from pandas.core.frame import DataFrame
 import pandas.core.nanops as nanops
 
-def convert_to_annual(series, freq=None):
+def pivot_annual(series, freq=None):
     """
     Group a series by years, taking leap years into account.