BUG: fix infer frequency for business daily (pandas-dev#16683)

Author: AdamGleave

asv_bench/benchmarks/timeseries.py

@@ -53,7 +53,11 @@ def setup(self):
         self.rng6 = date_range(start='1/1/1', periods=self.N, freq='B')
 
         self.rng7 = date_range(start='1/1/1700', freq='D', periods=100000)
-        self.a = self.rng7[:50000].append(self.rng7[50002:])
+        self.no_freq = self.rng7[:50000].append(self.rng7[50002:])
+        self.d_freq = self.rng7[:50000].append(self.rng7[50000:])
+
+        self.rng8 = date_range(start='1/1/1700', freq='B', periods=100000)
+        self.b_freq = self.rng8[:50000].append(self.rng8[50000:])
 
     def time_add_timedelta(self):
         (self.rng + dt.timedelta(minutes=2))
@@ -94,8 +98,14 @@ def time_infer_dst(self):
     def time_timeseries_is_month_start(self):
         self.rng6.is_month_start
 
-    def time_infer_freq(self):
-        infer_freq(self.a)
+    def time_infer_freq_none(self):
+        infer_freq(self.no_freq)
+
+    def time_infer_freq_daily(self):
+        infer_freq(self.d_freq)
+
+    def time_infer_freq_business(self):
+        infer_freq(self.b_freq)
 
 
 class TimeDatetimeConverter(object):

doc/source/whatsnew/v0.20.3.txt

@@ -1,55 +1,33 @@
 .. _whatsnew_0203:
 
-v0.20.3 (June ??, 2017)
+v0.20.3 (July 7, 2017)
 -----------------------
 
-This is a minor bug-fix release in the 0.20.x series and includes some small regression fixes,
-bug fixes and performance improvements.
-We recommend that all users upgrade to this version.
+This is a minor bug-fix release in the 0.20.x series and includes some small regression fixes
+and bug fixes. We recommend that all users upgrade to this version.
 
 .. contents:: What's new in v0.20.3
     :local:
     :backlinks: none
 
-
-.. _whatsnew_0203.enhancements:
-
-Enhancements
-~~~~~~~~~~~~
-
-
-
-
-
-
-.. _whatsnew_0203.performance:
-
-Performance Improvements
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-
-
-
-
-
 .. _whatsnew_0203.bug_fixes:
 
 Bug Fixes
 ~~~~~~~~~
-- Fixed issue with dataframe scatter plot for categorical data that reports incorrect column key not found when categorical data is used for plotting (:issue:`16199`)
-- Fixed issue with :meth:`DataFrame.style` where element id's were not unique (:issue:`16780`)
+
+- Fixed an issue with scatter plot for categorical data that incorrectly raised a ``KeyError`` when categorical data is used for plotting (:issue:`16199`)
+- Fixed an issue with :meth:`DataFrame.style` where element ids were not unique (:issue:`16780`)
 - Fixed a pytest marker failing downstream packages' tests suites (:issue:`16680`)
-- Fixed compat with loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`)
+- Fixed loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`)
 - Fixed a bug in failing to compute rolling computations of a column-MultiIndexed ``DataFrame`` (:issue:`16789`, :issue:`16825`)
-- Bug in a DataFrame/Series with a ``TimedeltaIndex`` when slice indexing (:issue:`16637`)
-
+- Fixed a bug in a slicing DataFrame/Series that have a  ``TimedeltaIndex`` (:issue:`16637`)
 
 Conversion
 ^^^^^^^^^^
 
 - Bug in pickle compat prior to the v0.20.x series, when ``UTC`` is a timezone in a Series/DataFrame/Index (:issue:`16608`)
 - Bug in ``Series`` construction when passing a ``Series`` with ``dtype='category'`` (:issue:`16524`).
-- Bug in ``DataFrame.astype()`` when passing a ``Series`` as the ``dtype`` kwarg. (:issue:`16717`).
+- Bug in :meth:`DataFrame.astype` when passing a ``Series`` as the ``dtype`` kwarg. (:issue:`16717`).
 
 Indexing
 ^^^^^^^^
@@ -62,40 +40,21 @@ I/O
 
 - Bug in :func:`read_csv` in which files weren't opened as binary files by the C engine on Windows, causing EOF characters mid-field, which would fail (:issue:`16039`, :issue:`16559`, :issue:`16675`)
 - Bug in :func:`read_hdf` in which reading a ``Series`` saved to an HDF file in 'fixed' format fails when an explicit ``mode='r'`` argument is supplied (:issue:`16583`)
-- Bug in :func:`DataFrame.to_latex` where ``bold_rows`` was wrongly specified to be ``True`` by default, whereas in reality row labels remained non-bold whatever parameter provided. (:issue:`16707`)
+- Bug in :meth:`DataFrame.to_latex` where ``bold_rows`` was wrongly specified to be ``True`` by default, whereas in reality row labels remained non-bold whatever parameter provided. (:issue:`16707`)
 
 Plotting
 ^^^^^^^^
-- Fix regression in series plotting that prevented RGB and RGBA tuples from being used as color arguments (:issue:`16233`)
-
-
-
-Groupby/Resample/Rolling
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-
-
-Sparse
-^^^^^^
-
-
 
+- Fixed regression that prevented RGB and RGBA tuples from being used as color arguments (:issue:`16233`)
 
 Reshaping
 ^^^^^^^^^
+
 - ``PeriodIndex`` / ``TimedeltaIndex.join`` was missing the ``sort=`` kwarg (:issue:`16541`) 
 - Bug in joining on a ``MultiIndex`` with a ``category`` dtype for a level (:issue:`16627`).
 - Bug in :func:`merge` when merging/joining with multiple categorical columns (:issue:`16767`)
 
-
-Numeric
-^^^^^^^
-
-
 Categorical
 ^^^^^^^^^^^
 
-- Bug in ``DataFrame.sort_values`` not respecting the ``kind`` with categorical data (:issue:`16793`)
-
-Other
-^^^^^
+- Bug in ``DataFrame.sort_values`` not respecting the ``kind`` parameter with categorical data (:issue:`16793`)

doc/source/whatsnew/v0.21.0.txt

@@ -169,7 +169,7 @@ Groupby/Resample/Rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 - Bug in ``DataFrame.resample().size()`` where an empty ``DataFrame`` did not return a ``Series`` (:issue:`14962`)
 
-
+- Bug in ``infer_freq`` causing indices with 2-day gaps during the working week to be wrongly inferred as business daily (:issue:`16624`)
 
 Sparse
 ^^^^^^

pandas/tests/indexes/timedeltas/test_timedelta.py

@@ -564,15 +564,23 @@ def test_freq_conversion(self):
 
 
 class TestSlicing(object):
+    @pytest.mark.parametrize('freq', ['B', 'D'])
+    def test_timedelta(self, freq):
+        index = date_range('1/1/2000', periods=50, freq=freq)
 
-    def test_timedelta(self):
-        # this is valid too
-        index = date_range('1/1/2000', periods=50, freq='B')
         shifted = index + timedelta(1)
         back = shifted + timedelta(-1)
-        assert tm.equalContents(index, back)
-        assert shifted.freq == index.freq
-        assert shifted.freq == back.freq
+        tm.assert_index_equal(index, back)
+
+        if freq == 'D':
+            expected = pd.tseries.offsets.Day(1)
+            assert index.freq == expected
+            assert shifted.freq == expected
+            assert back.freq == expected
+        else:  # freq == 'B'
+            assert index.freq == pd.tseries.offsets.BusinessDay(1)
+            assert shifted.freq is None
+            assert back.freq == pd.tseries.offsets.BusinessDay(1)
 
         result = index - timedelta(1)
         expected = index + timedelta(-1)

pandas/tests/tseries/test_frequencies.py

@@ -504,9 +504,14 @@ def test_raise_if_too_few(self):
         pytest.raises(ValueError, frequencies.infer_freq, index)
 
     def test_business_daily(self):
-        index = _dti(['12/31/1998', '1/3/1999', '1/4/1999'])
+        index = _dti(['01/01/1999', '1/4/1999', '1/5/1999'])
         assert frequencies.infer_freq(index) == 'B'
 
+    def test_business_daily_look_alike(self):
+        # GH 16624, do not infer 'B' when 'weekend' (2-day gap) in wrong place
+        index = _dti(['12/31/1998', '1/3/1999', '1/4/1999'])
+        assert frequencies.infer_freq(index) is None
+
     def test_day(self):
         self._check_tick(timedelta(1), 'D')
 

pandas/tseries/frequencies.py

@@ -975,8 +975,7 @@ def _infer_daily_rule(self):
             else:
                 return _maybe_add_count('D', days)
 
-        # Business daily. Maybe
-        if self.day_deltas == [1, 3]:
+        if self._is_business_daily():
             return 'B'
 
         wom_rule = self._get_wom_rule()
@@ -1012,6 +1011,19 @@ def _get_monthly_rule(self):
         return {'cs': 'MS', 'bs': 'BMS',
                 'ce': 'M', 'be': 'BM'}.get(pos_check)
 
+    def _is_business_daily(self):
+        # quick check: cannot be business daily
+        if self.day_deltas != [1, 3]:
+            return False
+
+        # probably business daily, but need to confirm
+        first_weekday = self.index[0].weekday()
+        shifts = np.diff(self.index.asi8)
+        shifts = np.floor_divide(shifts, _ONE_DAY)
+        weekdays = np.mod(first_weekday + np.cumsum(shifts), 7)
+        return np.all(((weekdays == 0) & (shifts == 3)) |
+                      ((weekdays > 0) & (weekdays <= 4) & (shifts == 1)))
+
     def _get_wom_rule(self):
         #         wdiffs = unique(np.diff(self.index.week))
         # We also need -47, -49, -48 to catch index spanning year boundary