Merge remote-tracking branch 'upstream/master' into skip_if_no

Author: simonjayhawkins

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://pandas.pydata.org/donate.html

asv_bench/benchmarks/index_object.py

@@ -188,7 +188,7 @@ def time_get_loc(self):
 
 class IntervalIndexMethod:
     # GH 24813
-    params = [10**3, 10**5, 10**7]
+    params = [10**3, 10**5]
 
     def setup(self, N):
         left = np.append(np.arange(N), np.array(0))

ci/deps/travis-36-cov.yaml

@@ -20,7 +20,8 @@ dependencies:
   # https://github.com/pydata/pandas-gbq/issues/271
   - google-cloud-bigquery<=1.11
   - psycopg2
-  - pyarrow=0.9.0
+  # pyarrow segfaults on load: https://github.com/pandas-dev/pandas/issues/26716
+  # - pyarrow=0.9.0
   - pymysql
   - pytables
   - python-snappy

ci/deps/travis-36-doc.yaml

@@ -33,7 +33,8 @@ dependencies:
   - pytz
   - scipy
   - seaborn
-  - sphinx
+  # recursion error with sphinx 2.1.0. https://github.com/pandas-dev/pandas/issues/26723
+  - sphinx==2.0.1
   - sqlalchemy
   - statsmodels
   - xarray

doc/source/whatsnew/v0.25.0.rst

@@ -513,6 +513,7 @@ Performance Improvements
 - Improved performance of :meth:`IntervalIndex.intersection` (:issue:`24813`)
 - Improved performance of :meth:`read_csv` by faster concatenating date columns without extra conversion to string for integer/float zero and float ``NaN``; by faster checking the string for the possibility of being a date (:issue:`25754`)
 - Improved performance of :attr:`IntervalIndex.is_unique` by removing conversion to ``MultiIndex`` (:issue:`24813`)
+- Restored performance of :meth:`DatetimeIndex.__iter__` by re-enabling specialized code path (:issue:`26702`)
 
 .. _whatsnew_0250.bug_fixes:
 
@@ -539,13 +540,15 @@ Datetimelike
 - Bug in adding :class:`DateOffset` with nonzero month to :class:`DatetimeIndex` would raise ``ValueError`` (:issue:`26258`)
 - Bug in :func:`to_datetime` which raises unhandled ``OverflowError`` when called with mix of invalid dates and ``NaN`` values with ``format='%Y%m%d'`` and ``error='coerce'`` (:issue:`25512`)
 - Bug in :func:`to_datetime` which raises ``TypeError`` for ``format='%Y%m%d'`` when called for invalid integer dates with length >= 6 digits with ``errors='ignore'``
+- Bug when comparing a :class:`PeriodIndex` against a zero-dimensional numpy array (:issue:`26689`)
 
 Timedelta
 ^^^^^^^^^
 
 - Bug in :func:`TimedeltaIndex.intersection` where for non-monotonic indices in some cases an empty ``Index`` was returned when in fact an intersection existed (:issue:`25913`)
 - Bug with comparisons between :class:`Timedelta` and ``NaT`` raising ``TypeError`` (:issue:`26039`)
 - Bug when adding or subtracting a :class:`BusinessHour` to a :class:`Timestamp` with the resulting time landing in a following or prior day respectively (:issue:`26381`)
+- Bug when comparing a :class:`TimedeltaIndex` against a zero-dimensional numpy array (:issue:`26689`)
 
 Timezones
 ^^^^^^^^^

environment.yml

@@ -16,14 +16,13 @@ dependencies:
   - cython>=0.28.2
 
   # code checks
+  - cpplint
   - flake8
   - flake8-comprehensions  # used by flake8, linting of unnecessary comprehensions
   - flake8-rst>=0.6.0,<=0.7.0  # linting of code blocks in rst files
   - isort  # check that imports are in the right order
   - mypy
   - pycodestyle  # used by flake8
-  - pip:
-    - cpplint
 
   # documentation
   - gitpython  # obtain contributors from git for whatsnew

pandas/core/arrays/period.py

@@ -4,6 +4,7 @@
 
 import numpy as np
 
+from pandas._libs import lib
 from pandas._libs.tslibs import (
     NaT, NaTType, frequencies as libfrequencies, iNaT, period as libperiod)
 from pandas._libs.tslibs.fields import isleapyear_arr
@@ -51,6 +52,7 @@ def _period_array_cmp(cls, op):
     def wrapper(self, other):
         op = getattr(self.asi8, opname)
 
+        other = lib.item_from_zerodim(other)
         if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
             return NotImplemented
 

pandas/core/arrays/timedeltas.py

@@ -62,6 +62,7 @@ def _td_array_cmp(cls, op):
     nat_result = opname == '__ne__'
 
     def wrapper(self, other):
+        other = lib.item_from_zerodim(other)
         if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
             return NotImplemented
 

pandas/core/indexes/datetimes.py

@@ -243,6 +243,8 @@ def _join_i8_wrapper(joinf, **kwargs):
     _is_numeric_dtype = False
     _infer_as_myclass = True
 
+    # Use faster implementation given we know we have DatetimeArrays
+    __iter__ = DatetimeArray.__iter__
     # some things like freq inference make use of these attributes.
     _bool_ops = DatetimeArray._bool_ops
     _object_ops = DatetimeArray._object_ops

pandas/core/indexes/range.py

@@ -346,8 +346,9 @@ def __contains__(self, key: Union[int, np.integer]) -> bool:
     @Appender(_index_shared_docs['get_loc'])
     def get_loc(self, key, method=None, tolerance=None):
         if is_integer(key) and method is None and tolerance is None:
+            new_key = int(key)
             try:
-                return self._range.index(key)
+                return self._range.index(new_key)
             except ValueError:
                 raise KeyError(key)
         return super().get_loc(key, method=method, tolerance=tolerance)
@@ -608,19 +609,15 @@ def __getitem__(self, key):
         """
         Conserve RangeIndex type for scalar and slice keys.
         """
-        super_getitem = super().__getitem__
-
         if is_scalar(key):
             if not lib.is_integer(key):
                 raise IndexError("only integers, slices (`:`), "
                                  "ellipsis (`...`), numpy.newaxis (`None`) "
                                  "and integer or boolean "
                                  "arrays are valid indices")
-            n = com.cast_scalar_indexer(key)
-            if n != key:
-                return super_getitem(key)
+            new_key = int(key)
             try:
-                return self._range[key]
+                return self._range[new_key]
             except IndexError:
                 raise IndexError("index {key} is out of bounds for axis 0 "
                                  "with size {size}".format(key=key,
@@ -630,7 +627,7 @@ def __getitem__(self, key):
             return self.from_range(new_range, name=self.name)
 
         # fall back to Int64Index
-        return super_getitem(key)
+        return super().__getitem__(key)
 
     def __floordiv__(self, other):
         if isinstance(other, (ABCSeries, ABCDataFrame)):

pandas/tests/arithmetic/test_datetime64.py

@@ -37,6 +37,7 @@ def assert_all(obj):
 # ------------------------------------------------------------------
 # Comparisons
 
+
 class TestDatetime64DataFrameComparison:
     @pytest.mark.parametrize('timestamps', [
         [pd.Timestamp('2012-01-01 13:00:00+00:00')] * 2,
@@ -338,6 +339,17 @@ def test_comparison_tzawareness_compat(self, op):
 
 class TestDatetimeIndexComparisons:
 
+    # TODO: parametrize over box
+    def test_compare_zerodim(self, tz_naive_fixture):
+        # Test comparison with zero-dimensional array is unboxed
+        tz = tz_naive_fixture
+        dti = date_range('20130101', periods=3, tz=tz)
+
+        other = np.array(dti.to_numpy()[0])
+        result = dti <= other
+        expected = np.array([True, False, False])
+        tm.assert_numpy_array_equal(result, expected)
+
     # TODO: moved from tests.indexes.test_base; parametrize and de-duplicate
     @pytest.mark.parametrize("op", [
         operator.eq, operator.ne, operator.gt, operator.lt,

pandas/tests/arithmetic/test_period.py

@@ -22,6 +22,16 @@
 
 class TestPeriodIndexComparisons:
 
+    # TODO: parameterize over boxes
+    def test_compare_zerodim(self):
+        # GH#26689 make sure we unbox zero-dimensional arrays
+        pi = pd.period_range('2000', periods=4)
+        other = np.array(pi.to_numpy()[0])
+
+        result = pi <= other
+        expected = np.array([True, False, False, False])
+        tm.assert_numpy_array_equal(result, expected)
+
     @pytest.mark.parametrize("other", ["2017", 2017])
     def test_eq(self, other):
         idx = PeriodIndex(['2017', '2017', '2018'], freq="D")

pandas/tests/arithmetic/test_timedelta64.py

@@ -33,6 +33,19 @@ def get_upcast_box(box, vector):
 class TestTimedelta64ArrayComparisons:
     # TODO: All of these need to be parametrized over box
 
+    def test_compare_timedelta64_zerodim(self):
+        # GH#26689 should unbox when comparing with zerodim array
+        tdi = pd.timedelta_range('2H', periods=4)
+        other = np.array(tdi.to_numpy()[0])
+
+        res = tdi <= other
+        expected = np.array([True, False, False, False])
+        tm.assert_numpy_array_equal(res, expected)
+
+        with pytest.raises(TypeError):
+            # zero-dim of wrong dtype should still raise
+            tdi >= np.array(4)
+
     def test_compare_timedelta_series(self):
         # regresssion test for GH#5963
         s = pd.Series([timedelta(days=1), timedelta(days=2)])
@@ -1524,7 +1537,8 @@ def test_td64arr_div_nat_invalid(self, box_with_array):
         rng = timedelta_range('1 days', '10 days', name='foo')
         rng = tm.box_expected(rng, box_with_array)
 
-        with pytest.raises(TypeError, match='true_divide cannot use operands'):
+        with pytest.raises(TypeError,
+                           match="'?true_divide'? cannot use operands"):
             rng / pd.NaT
         with pytest.raises(TypeError, match='Cannot divide NaTType by'):
             pd.NaT / rng

pandas/tests/indexes/timedeltas/test_arithmetic.py

@@ -218,7 +218,7 @@ def test_ops_ndarray(self):
         expected = pd.to_timedelta(['2 days']).values
         tm.assert_numpy_array_equal(td * np.array([2]), expected)
         tm.assert_numpy_array_equal(np.array([2]) * td, expected)
-        msg = ("ufunc multiply cannot use operands with types"
+        msg = ("ufunc '?multiply'? cannot use operands with types"
                r" dtype\('<m8\[ns\]'\) and dtype\('<m8\[ns\]'\)")
         with pytest.raises(TypeError, match=msg):
             td * other

pandas/tests/plotting/common.py

@@ -36,6 +36,7 @@ def setup_method(self, method):
 
         self.mpl_ge_2_2_3 = plotting._compat._mpl_ge_2_2_3()
         self.mpl_ge_3_0_0 = plotting._compat._mpl_ge_3_0_0()
+        self.mpl_ge_3_1_0 = plotting._compat._mpl_ge_3_1_0()
 
         self.bp_n_objects = 7
         self.polycollection_factor = 2
@@ -440,10 +441,18 @@ def _check_grid_settings(self, obj, kinds, kws={}):
         import matplotlib as mpl
 
         def is_grid_on():
-            xoff = all(not g.gridOn
-                       for g in self.plt.gca().xaxis.get_major_ticks())
-            yoff = all(not g.gridOn
-                       for g in self.plt.gca().yaxis.get_major_ticks())
+            xticks = self.plt.gca().xaxis.get_major_ticks()
+            yticks = self.plt.gca().yaxis.get_major_ticks()
+            # for mpl 2.2.2, gridOn and gridline.get_visible disagree.
+            # for new MPL, they are the same.
+
+            if self.mpl_ge_3_0_0:
+                xoff = all(not g.gridline.get_visible() for g in xticks)
+                yoff = all(not g.gridline.get_visible() for g in yticks)
+            else:
+                xoff = all(not g.gridOn for g in xticks)
+                yoff = all(not g.gridOn for g in yticks)
+
             return not (xoff and yoff)
 
         spndx = 1

pandas/tests/plotting/test_frame.py

@@ -383,8 +383,10 @@ def test_subplots(self):
             for ax in axes[:-2]:
                 self._check_visible(ax.xaxis)  # xaxis must be visible for grid
                 self._check_visible(ax.get_xticklabels(), visible=False)
-                self._check_visible(
-                    ax.get_xticklabels(minor=True), visible=False)
+                if not (kind == 'bar' and self.mpl_ge_3_1_0):
+                    # change https://github.com/pandas-dev/pandas/issues/26714
+                    self._check_visible(
+                        ax.get_xticklabels(minor=True), visible=False)
                 self._check_visible(ax.xaxis.get_label(), visible=False)
                 self._check_visible(ax.get_yticklabels())
 
@@ -800,7 +802,10 @@ def test_line_area_stacked(self):
                 with pytest.raises(ValueError):
                     mixed_df.plot(stacked=True)
 
-                _check_plot_works(df.plot, kind=kind, logx=True, stacked=True)
+                # Use an index with strictly positive values, preventing
+                #  matplotlib from warning about ignoring xlim
+                df2 = df.set_index(df.index + 1)
+                _check_plot_works(df2.plot, kind=kind, logx=True, stacked=True)
 
     def test_line_area_nan_df(self):
         values1 = [1, 2, np.nan, 3]

requirements-dev.txt

@@ -3,13 +3,13 @@ python-dateutil>=2.5.0
 pytz
 asv
 cython>=0.28.2
+cpplint
 flake8
 flake8-comprehensions
 flake8-rst>=0.6.0,<=0.7.0
 isort
 mypy
 pycodestyle
-cpplint
 gitpython
 sphinx
 numpydoc>=0.9.0