ENH: nanosecond tweak per #2170. period spanning in upsampling

Author: wesm

RELEASE.rst

@@ -34,6 +34,13 @@ pandas 0.9.1
   - Add `where` and `mask` functions to DataFrame (#2109, #2151)
   - Add `at_time` and `between_time` functions to DataFrame (#2149)
 
+**API Changes**
+
+  - Upsampling period index "spans" intervals. Example: annual periods
+    upsampled to monthly will span all months in each year
+  - Period.end_time will yield timestamp at last nanosecond in the interval
+    (#2124, #2125, #1764)
+
 **Improvements to existing features**
 
   - Time rule inference for week-of-month (e.g. WOM-2FRI) rules (#2140)

pandas/core/generic.py

@@ -200,7 +200,7 @@ def between_time(self, start_time, end_time, include_start=True,
             raise TypeError('Index must be DatetimeIndex')
 
     def resample(self, rule, how=None, axis=0, fill_method=None,
-                 closed='right', label='right', convention=None,
+                 closed='right', label='right', convention='start',
                  kind=None, loffset=None, limit=None, base=0):
         """
         Convenience method for frequency conversion and resampling of regular

pandas/tseries/period.py

@@ -181,11 +181,12 @@ def asfreq(self, freq, how='E'):
 
     @property
     def start_time(self):
-        return self.to_timestamp('s', how='S')
+        return self.to_timestamp(how='S')
 
     @property
     def end_time(self):
-        return self.to_timestamp('s', how='E')
+        ordinal = (self + 1).start_time.value - 1
+        return Timestamp(ordinal)
 
     def to_timestamp(self, freq=None, how='start'):
         """

pandas/tseries/resample.py

@@ -219,7 +219,8 @@ def _resample_periods(self, obj):
             return obj.reindex(new_index)
         else:
             start = axlabels[0].asfreq(self.freq, how=self.convention)
-            end = axlabels[-1].asfreq(self.freq, how=self.convention)
+            end = axlabels[-1].asfreq(self.freq, how='end')
+
             new_index = period_range(start, end, freq=self.freq)
 
         # Start vs. end of period

pandas/tseries/tests/test_period.py

@@ -224,14 +224,18 @@ def test_to_timestamp(self):
 
         from_lst = ['A', 'Q', 'M', 'W', 'B',
                     'D', 'H', 'Min', 'S']
+
+        def _ex(p):
+            return Timestamp((p + 1).start_time.value - 1)
+
         for i, fcode in enumerate(from_lst):
             p = Period('1982', freq=fcode)
             result = p.to_timestamp().to_period(fcode)
             self.assertEquals(result, p)
 
             self.assertEquals(p.start_time, p.to_timestamp(how='S'))
 
-            self.assertEquals(p.end_time, p.to_timestamp('s', how='E'))
+            self.assertEquals(p.end_time, _ex(p))
 
         # Frequency other than daily
 
@@ -272,30 +276,34 @@ def test_start_time(self):
 
     def test_end_time(self):
         p = Period('2012', freq='A')
-        xp = datetime(2012, 12, 31, 23, 59, 59)
+
+        def _ex(*args):
+            return Timestamp(Timestamp(datetime(*args)).value - 1)
+
+        xp = _ex(2013, 1, 1)
         self.assertEquals(xp, p.end_time)
 
         p = Period('2012', freq='Q')
-        xp = datetime(2012, 3, 31, 23, 59, 59)
+        xp = _ex(2012, 4, 1)
         self.assertEquals(xp, p.end_time)
 
         p = Period('2012', freq='M')
-        xp = datetime(2012, 1, 31, 23, 59, 59)
+        xp = _ex(2012, 2, 1)
         self.assertEquals(xp, p.end_time)
 
-        xp = datetime(2012, 1, 1, 23, 59, 59)
+        xp = _ex(2012, 1, 2)
         p = Period('2012', freq='D')
         self.assertEquals(p.end_time, xp)
 
-        xp = datetime(2012, 1, 1, 0, 59, 59)
+        xp = _ex(2012, 1, 1, 1)
         p = Period('2012', freq='H')
         self.assertEquals(p.end_time, xp)
 
-        self.assertEquals(Period('2012', freq='B').end_time,
-                          datetime(2011, 12, 30, 23, 59, 59))
+        xp = _ex(2012, 1, 2)
+        self.assertEquals(Period('2012', freq='B').end_time, xp)
 
-        self.assertEquals(Period('2012', freq='W').end_time,
-                          datetime(2012, 1, 1, 23, 59, 59))
+        xp = _ex(2012, 1, 2)
+        self.assertEquals(Period('2012', freq='W').end_time, xp)
 
 
     def test_properties_annually(self):

pandas/tseries/tests/test_resample.py

@@ -451,24 +451,26 @@ def test_resample_anchored_intraday(self):
         df = DataFrame(rng.month, index=rng)
 
         result = df.resample('M')
-        expected = df.resample('M', kind='period').to_timestamp()
+        expected = df.resample('M', kind='period').to_timestamp(how='end')
         tm.assert_frame_equal(result, expected)
 
         result = df.resample('M', closed='left')
-        exp = df.tshift(1, freq='D').resample('M', kind='period').to_timestamp()
+        exp = df.tshift(1, freq='D').resample('M', kind='period')
+        exp = exp.to_timestamp(how='end')
+
         tm.assert_frame_equal(result, exp)
 
         rng = date_range('1/1/2012', '4/1/2013', freq='10min')
         df = DataFrame(rng.month, index=rng)
 
         result = df.resample('Q')
-        expected = df.resample('Q', kind='period').to_timestamp()
+        expected = df.resample('Q', kind='period').to_timestamp(how='end')
         tm.assert_frame_equal(result, expected)
 
         result = df.resample('Q', closed='left')
         expected = df.tshift(1, freq='D').resample('Q', kind='period',
                                                    closed='left')
-        expected = expected.to_timestamp()
+        expected = expected.to_timestamp(how='end')
         tm.assert_frame_equal(result, expected)
 
         ts = _simple_ts('2012-04-29 23:00', '2012-04-30 5:00', freq='h')
@@ -622,7 +624,8 @@ def test_upsample_with_limit(self):
         rng = period_range('1/1/2000', periods=5, freq='A')
         ts = Series(np.random.randn(len(rng)), rng)
 
-        result = ts.resample('M', fill_method='ffill', limit=2)
+        result = ts.resample('M', fill_method='ffill', limit=2,
+                             convention='end')
         expected = ts.asfreq('M').reindex(result.index, method='ffill',
                                           limit=2)
         assert_series_equal(result, expected)
@@ -646,6 +649,17 @@ def test_annual_upsample(self):
         exp = df['a'].resample('D', fill_method='ffill')
         assert_series_equal(rdf['a'], exp)
 
+
+        rng = period_range('2000', '2003', freq='A-DEC')
+        ts = Series([1, 2, 3, 4], index=rng)
+
+        result = ts.resample('M', fill_method='ffill')
+        ex_index = period_range('2000-01', '2003-12', freq='M')
+
+        expected = ts.asfreq('M', how='start').reindex(ex_index,
+                                                       method='ffill')
+        assert_series_equal(result, expected)
+
     def test_quarterly_upsample(self):
         targets = ['D', 'B', 'M']
 
@@ -696,8 +710,9 @@ def test_resample_to_quarterly(self):
             ts = _simple_pts('1990', '1992', freq='A-%s' % month)
             quar_ts = ts.resample('Q-%s' % month, fill_method='ffill')
 
-            stamps = ts.to_timestamp('D', how='end')
-            qdates = period_range(stamps.index[0], stamps.index[-1],
+            stamps = ts.to_timestamp('D', how='start')
+            qdates = period_range(ts.index[0].asfreq('D', 'start'),
+                                  ts.index[-1].asfreq('D', 'end'),
                                   freq='Q-%s' % month)
 
             expected = stamps.reindex(qdates.to_timestamp('D', 'e'),
@@ -711,9 +726,13 @@ def test_resample_to_quarterly(self):
 
         for how in ['start', 'end']:
             result = ts.resample('Q-MAR', convention=how, fill_method='ffill')
-            expected = ts.asfreq('Q-MAR', how=how).to_timestamp('D')
-            expected = expected.resample('Q-MAR', fill_method='ffill')
-            assert_series_equal(result, expected.to_period('Q-MAR'))
+            expected = ts.asfreq('Q-MAR', how=how)
+            expected = expected.reindex(result.index, method='ffill')
+
+            # .to_timestamp('D')
+            # expected = expected.resample('Q-MAR', fill_method='ffill')
+
+            assert_series_equal(result, expected)
 
     def test_resample_fill_missing(self):
         rng = PeriodIndex([2000, 2005, 2007, 2009], freq='A')
@@ -752,7 +771,7 @@ def test_upsample_daily_business_daily(self):
 
         ts = _simple_pts('1/1/2000', '2/1/2000')
         result = ts.resample('H', convention='s')
-        exp_rng = period_range('1/1/2000', '2/1/2000', freq='H')
+        exp_rng = period_range('1/1/2000', '2/1/2000 23:00', freq='H')
         expected = ts.asfreq('H', how='s').reindex(exp_rng)
         assert_series_equal(result, expected)