Merge branch 'master' into bug_groupby_quantile_arraylike_fails

Author: fujiaxiang

ci/code_checks.sh

@@ -122,13 +122,18 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # Check for imports from collections.abc instead of `from collections import abc`
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from pandas.core import common" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from collections.abc import" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from numpy import nan" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Checks for test suite
     # Check for imports from pandas.util.testing instead of `import pandas.util.testing as tm`
     invgrep -R --include="*.py*" -E "from pandas.util.testing import" pandas/tests
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
     invgrep -R --include="*.py*" -E "from pandas.util import testing as tm" pandas/tests
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
@@ -195,6 +200,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' .
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
+    invgrep -R --include=*.{py,pyx} '!r}' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check for use of comment-based annotation syntax' ; echo $MSG
     invgrep -R --include="*.py" -P '# type: (?!ignore)' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"

doc/source/user_guide/advanced.rst

@@ -573,7 +573,7 @@ When working with an ``Index`` object directly, rather than via a ``DataFrame``,
    .. code-block:: none
 
       >>> mi.levels[0].name = 'name via level'
-      >>> mi.names[0]  # only works for older panads
+      >>> mi.names[0]  # only works for older pandas
       'name via level'
 
    As of pandas 1.0, this will *silently* fail to update the names

doc/source/user_guide/missing_data.rst

@@ -791,7 +791,7 @@ the nullable :doc:`integer <integer_na>`, boolean and
 :ref:`dedicated string <text.types>` data types as the missing value indicator.
 
 The goal of ``pd.NA`` is provide a "missing" indicator that can be used
-consistently accross data types (instead of ``np.nan``, ``None`` or ``pd.NaT``
+consistently across data types (instead of ``np.nan``, ``None`` or ``pd.NaT``
 depending on the data type).
 
 For example, when having missing values in a Series with the nullable integer

doc/source/user_guide/text.rst

@@ -101,10 +101,10 @@ l. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
 2. Some string methods, like :meth:`Series.str.decode` are not available
    on ``StringArray`` because ``StringArray`` only holds strings, not
    bytes.
-3. In comparision operations, :class:`arrays.StringArray` and ``Series`` backed
+3. In comparison operations, :class:`arrays.StringArray` and ``Series`` backed
    by a ``StringArray`` will return an object with :class:`BooleanDtype`,
    rather than a ``bool`` dtype object. Missing values in a ``StringArray``
-   will propagate in comparision operations, rather than always comparing
+   will propagate in comparison operations, rather than always comparing
    unequal like :attr:`numpy.nan`.
 
 Everything else that follows in the rest of this document applies equally to

doc/source/whatsnew/v1.0.0.rst

@@ -111,7 +111,7 @@ A new ``pd.NA`` value (singleton) is introduced to represent scalar missing
 values. Up to now, ``np.nan`` is used for this for float data, ``np.nan`` or
 ``None`` for object-dtype data and ``pd.NaT`` for datetime-like data. The
 goal of ``pd.NA`` is provide a "missing" indicator that can be used
-consistently accross data types. For now, the nullable integer and boolean
+consistently across data types. For now, the nullable integer and boolean
 data types and the new string data type make use of ``pd.NA`` (:issue:`28095`).
 
 .. warning::
@@ -571,6 +571,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - A tuple passed to :meth:`DataFrame.groupby` is now exclusively treated as a single key (:issue:`18314`)
 - Removed the previously deprecated :meth:`Index.contains`, use ``key in index`` instead (:issue:`30103`)
 - Addition and subtraction of ``int`` or integer-arrays is no longer allowed in :class:`Timestamp`, :class:`DatetimeIndex`, :class:`TimedeltaIndex`, use ``obj + n * obj.freq`` instead of ``obj + n`` (:issue:`22535`)
+- Removed :meth:`Series.ptp` (:issue:`21614`)
 - Removed :meth:`Series.from_array` (:issue:`18258`)
 - Removed :meth:`DataFrame.from_items` (:issue:`18458`)
 - Removed :meth:`DataFrame.as_matrix`, :meth:`Series.as_matrix` (:issue:`18458`)
@@ -716,8 +717,10 @@ Datetimelike
 - Bug in :func:`pandas.to_datetime` failing for `deques` when using ``cache=True`` (the default) (:issue:`29403`)
 - Bug in :meth:`Series.item` with ``datetime64`` or ``timedelta64`` dtype, :meth:`DatetimeIndex.item`, and :meth:`TimedeltaIndex.item` returning an integer instead of a :class:`Timestamp` or :class:`Timedelta` (:issue:`30175`)
 - Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`)
+- Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`)
 - Bug in :meth:`DataFrame.append` would remove the timezone-awareness of new data (:issue:`30238`)
 
+
 Timedelta
 ^^^^^^^^^
 - Bug in subtracting a :class:`TimedeltaIndex` or :class:`TimedeltaArray` from a ``np.datetime64`` object (:issue:`29558`)
@@ -825,7 +828,7 @@ Plotting
 - Bug where :meth:`DataFrame.boxplot` would not accept a `color` parameter like `DataFrame.plot.box` (:issue:`26214`)
 - Bug in the ``xticks`` argument being ignored for :meth:`DataFrame.plot.bar` (:issue:`14119`)
 - :func:`set_option` now validates that the plot backend provided to ``'plotting.backend'`` implements the backend when the option is set, rather than when a plot is created (:issue:`28163`)
-- :meth:`DataFrame.plot` now allow a ``backend`` keyword arugment to allow changing between backends in one session (:issue:`28619`).
+- :meth:`DataFrame.plot` now allow a ``backend`` keyword argument to allow changing between backends in one session (:issue:`28619`).
 - Bug in color validation incorrectly raising for non-color styles (:issue:`29122`).
 
 Groupby/resample/rolling

environment.yml

@@ -33,7 +33,8 @@ dependencies:
   - nbconvert>=5.4.1
   - nbsphinx
   - pandoc
-  # Dask and its dependencies
+
+  # Dask and its dependencies (that dont install with dask)
   - dask-core
   - toolz>=0.7.3
   - fsspec>=0.5.1
@@ -54,6 +55,8 @@ dependencies:
   - pytest>=5.0.1
   - pytest-cov
   - pytest-xdist>=1.21
+
+  # downstream tests
   - seaborn
   - statsmodels
 
@@ -74,22 +77,28 @@ dependencies:
   - scipy>=1.1
 
   # optional for io
-  - beautifulsoup4>=4.6.0  # pandas.read_html
+  # ---------------
+  # pd.read_html
+  - beautifulsoup4>=4.6.0
+  - html5lib
+  - lxml
+
+  # pd.read_excel, DataFrame.to_excel, pd.ExcelWriter, pd.ExcelFile
+  - openpyxl<=3.0.1
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - odfpy
+
   - fastparquet>=0.3.2  # pandas.read_parquet, DataFrame.to_parquet
-  - html5lib  # pandas.read_html
-  - lxml  # pandas.read_html
-  - openpyxl<=3.0.1  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
   - pyarrow>=0.13.1  # pandas.read_parquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather
+  - python-snappy  # required by pyarrow
+
   - pyqt>=5.9.2  # pandas.read_clipboard
   - pytables>=3.4.2  # pandas.read_hdf, DataFrame.to_hdf
-  - python-snappy  # required by pyarrow
   - s3fs  # pandas.read_csv... when using 's3://...' path
   - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
   - xarray  # DataFrame.to_xarray
-  - xlrd  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
-  - xlsxwriter  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
-  - xlwt  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
-  - odfpy  # pandas.read_excel
   - pyreadstat  # pandas.read_spss
   - pip:
     - git+https://github.com/pandas-dev/pandas-sphinx-theme.git@master

pandas/_libs/intervaltree.pxi.in

@@ -114,43 +114,6 @@ cdef class IntervalTree(IntervalMixin):
         sort_order = np.lexsort(values)
         return is_monotonic(sort_order, False)[0]
 
-    def get_loc(self, scalar_t key):
-        """Return all positions corresponding to intervals that overlap with
-        the given scalar key
-        """
-        result = Int64Vector()
-        self.root.query(result, key)
-        if not result.data.n:
-            raise KeyError(key)
-        return result.to_array().astype('intp')
-
-    def _get_partial_overlap(self, key_left, key_right, side):
-        """Return all positions corresponding to intervals with the given side
-        falling between the left and right bounds of an interval query
-        """
-        if side == 'left':
-            values = self.left
-            sorter = self.left_sorter
-        else:
-            values = self.right
-            sorter = self.right_sorter
-        key = [key_left, key_right]
-        i, j = values.searchsorted(key, sorter=sorter)
-        return sorter[i:j]
-
-    def get_loc_interval(self, key_left, key_right):
-        """Lookup the intervals enclosed in the given interval bounds
-
-        The given interval is presumed to have closed bounds.
-        """
-        import pandas as pd
-        left_overlap = self._get_partial_overlap(key_left, key_right, 'left')
-        right_overlap = self._get_partial_overlap(key_left, key_right, 'right')
-        enclosing = self.get_loc(0.5 * (key_left + key_right))
-        combined = np.concatenate([left_overlap, right_overlap, enclosing])
-        uniques = pd.unique(combined)
-        return uniques.astype('intp')
-
     def get_indexer(self, scalar_t[:] target):
         """Return the positions corresponding to unique intervals that overlap
         with the given array of scalar targets.

pandas/core/arrays/datetimelike.py

@@ -915,10 +915,8 @@ def _is_unique(self):
     __rdivmod__ = make_invalid_op("__rdivmod__")
 
     def _add_datetimelike_scalar(self, other):
-        # Overriden by TimedeltaArray
-        raise TypeError(
-            f"cannot add {type(self).__name__} and " f"{type(other).__name__}"
-        )
+        # Overridden by TimedeltaArray
+        raise TypeError(f"cannot add {type(self).__name__} and {type(other).__name__}")
 
     _add_datetime_arraylike = _add_datetimelike_scalar
 
@@ -930,7 +928,7 @@ def _sub_datetimelike_scalar(self, other):
     _sub_datetime_arraylike = _sub_datetimelike_scalar
 
     def _sub_period(self, other):
-        # Overriden by PeriodArray
+        # Overridden by PeriodArray
         raise TypeError(f"cannot subtract Period from a {type(self).__name__}")
 
     def _add_offset(self, offset):
@@ -1087,7 +1085,7 @@ def _addsub_int_array(self, other, op):
         -------
         result : same class as self
         """
-        # _addsub_int_array is overriden by PeriodArray
+        # _addsub_int_array is overridden by PeriodArray
         assert not is_period_dtype(self)
         assert op in [operator.add, operator.sub]
 

pandas/core/base.py

@@ -2,7 +2,6 @@
 Base and utility classes for pandas objects.
 """
 import builtins
-from collections import OrderedDict
 import textwrap
 from typing import Dict, FrozenSet, List, Optional
 
@@ -141,39 +140,35 @@ class SelectionMixin:
     _internal_names = ["_cache", "__setstate__"]
     _internal_names_set = set(_internal_names)
 
-    _builtin_table = OrderedDict(
-        ((builtins.sum, np.sum), (builtins.max, np.max), (builtins.min, np.min))
-    )
-
-    _cython_table = OrderedDict(
-        (
-            (builtins.sum, "sum"),
-            (builtins.max, "max"),
-            (builtins.min, "min"),
-            (np.all, "all"),
-            (np.any, "any"),
-            (np.sum, "sum"),
-            (np.nansum, "sum"),
-            (np.mean, "mean"),
-            (np.nanmean, "mean"),
-            (np.prod, "prod"),
-            (np.nanprod, "prod"),
-            (np.std, "std"),
-            (np.nanstd, "std"),
-            (np.var, "var"),
-            (np.nanvar, "var"),
-            (np.median, "median"),
-            (np.nanmedian, "median"),
-            (np.max, "max"),
-            (np.nanmax, "max"),
-            (np.min, "min"),
-            (np.nanmin, "min"),
-            (np.cumprod, "cumprod"),
-            (np.nancumprod, "cumprod"),
-            (np.cumsum, "cumsum"),
-            (np.nancumsum, "cumsum"),
-        )
-    )
+    _builtin_table = {builtins.sum: np.sum, builtins.max: np.max, builtins.min: np.min}
+
+    _cython_table = {
+        builtins.sum: "sum",
+        builtins.max: "max",
+        builtins.min: "min",
+        np.all: "all",
+        np.any: "any",
+        np.sum: "sum",
+        np.nansum: "sum",
+        np.mean: "mean",
+        np.nanmean: "mean",
+        np.prod: "prod",
+        np.nanprod: "prod",
+        np.std: "std",
+        np.nanstd: "std",
+        np.var: "var",
+        np.nanvar: "var",
+        np.median: "median",
+        np.nanmedian: "median",
+        np.max: "max",
+        np.nanmax: "max",
+        np.min: "min",
+        np.nanmin: "min",
+        np.cumprod: "cumprod",
+        np.nancumprod: "cumprod",
+        np.cumsum: "cumsum",
+        np.nancumsum: "cumsum",
+    }
 
     @property
     def _selection_name(self):
@@ -328,7 +323,7 @@ def _aggregate(self, arg, *args, **kwargs):
             # eg. {'A' : ['mean']}, normalize all to
             # be list-likes
             if any(is_aggregator(x) for x in arg.values()):
-                new_arg = OrderedDict()
+                new_arg = {}
                 for k, v in arg.items():
                     if not isinstance(v, (tuple, list, dict)):
                         new_arg[k] = [v]
@@ -386,16 +381,16 @@ def _agg_2dim(name, how):
             def _agg(arg, func):
                 """
                 run the aggregations over the arg with func
-                return an OrderedDict
+                return a dict
                 """
-                result = OrderedDict()
+                result = {}
                 for fname, agg_how in arg.items():
                     result[fname] = func(fname, agg_how)
                 return result
 
             # set the final keys
             keys = list(arg.keys())
-            result = OrderedDict()
+            result = {}
 
             if self._selection is not None:
 

pandas/core/dtypes/cast.py

@@ -820,9 +820,7 @@ def astype_nansafe(arr, dtype, copy: bool = True, skipna: bool = False):
         if dtype.kind == "M":
             return arr.astype(dtype)
 
-        raise TypeError(
-            f"cannot astype a datetimelike from [{arr.dtype}] " f"to [{dtype}]"
-        )
+        raise TypeError(f"cannot astype a datetimelike from [{arr.dtype}] to [{dtype}]")
 
     elif is_timedelta64_dtype(arr):
         if is_object_dtype(dtype):
@@ -842,9 +840,7 @@ def astype_nansafe(arr, dtype, copy: bool = True, skipna: bool = False):
         elif dtype == _TD_DTYPE:
             return arr.astype(_TD_DTYPE, copy=copy)
 
-        raise TypeError(
-            f"cannot astype a timedelta from [{arr.dtype}] " f"to [{dtype}]"
-        )
+        raise TypeError(f"cannot astype a timedelta from [{arr.dtype}] to [{dtype}]")
 
     elif np.issubdtype(arr.dtype, np.floating) and np.issubdtype(dtype, np.integer):