Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-misc

Author: jbrockmendel

.github/workflows/ci.yml

@@ -15,12 +15,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Setting conda path
+      run: echo "::add-path::${HOME}/miniconda3/bin"
+
     - name: Checkout
       uses: actions/checkout@v1
 
-    - name: Setting conda path
-      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
-
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
       if: true

ci/azure/posix.yml

@@ -44,13 +44,15 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
-        py37_np_dev:
-          ENV_FILE: ci/deps/azure-37-numpydev.yaml
-          CONDA_PY: "37"
-          PATTERN: "not slow and not network"
-          TEST_ARGS: "-W error"
-          PANDAS_TESTING_MODE: "deprecate"
-          EXTRA_APT: "xsel"
+        # Disabled for NumPy object-dtype warning.
+        # https://github.com/pandas-dev/pandas/issues/30043
+        # py37_np_dev:
+        #   ENV_FILE: ci/deps/azure-37-numpydev.yaml
+        #   CONDA_PY: "37"
+        #   PATTERN: "not slow and not network"
+        #   TEST_ARGS: "-W error"
+        #   PANDAS_TESTING_MODE: "deprecate"
+        #   EXTRA_APT: "xsel"
 
   steps:
     - script: |

ci/deps/azure-36-locale.yaml

@@ -27,6 +27,9 @@ dependencies:
   - xlrd=1.1.0
   - xlsxwriter=0.9.8
   - xlwt=1.2.0
+  # lowest supported version of pyarrow (putting it here instead of in
+  # azure-36-minimum_versions because it needs numpy >= 1.14)
+  - pyarrow=0.12
   - pip
   - pip:
     - html5lib==1.0b2

ci/deps/azure-37-locale.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.7.*
@@ -33,3 +32,4 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
+  - pyarrow>=0.15

ci/deps/travis-36-cov.yaml

@@ -27,13 +27,13 @@ dependencies:
   - numexpr
   - numpy=1.15.*
   - odfpy
-  - openpyxl
+  - openpyxl<=3.0.1
+  # https://github.com/pandas-dev/pandas/pull/30009 openpyxl 3.0.2 broke
   - pandas-gbq
   # https://github.com/pydata/pandas-gbq/issues/271
   - google-cloud-bigquery<=1.11
   - psycopg2
-  # pyarrow segfaults on load: https://github.com/pandas-dev/pandas/issues/26716
-  # - pyarrow=0.9.0
+  - pyarrow>=0.12.0
   - pymysql
   - pytables
   - python-snappy

ci/print_skipped.py

@@ -5,12 +5,12 @@
 
 def main(filename):
     if not os.path.isfile(filename):
-        return
+        raise RuntimeError(f"Could not find junit file {filename!r}")
 
     tree = et.parse(filename)
     root = tree.getroot()
     current_class = ""
-    for el in root.findall("testcase"):
+    for el in root.iter("testcase"):
         cn = el.attrib["classname"]
         for sk in el.findall("skipped"):
             old_class = current_class
@@ -32,8 +32,7 @@ def main(filename):
             print("-" * 80)
         else:
             print(
-                "#{i} {class_name}.{test_name}: {message}".format(
-                    **dict(test_data, i=i)
-                )
+                f"#{i} {test_data['class_name']}."
+                f"{test_data['test_name']}: {test_data['message']}"
             )
             i += 1

doc/redirects.csv

@@ -636,7 +636,6 @@ generated/pandas.Index.equals,../reference/api/pandas.Index.equals
 generated/pandas.Index.factorize,../reference/api/pandas.Index.factorize
 generated/pandas.Index.fillna,../reference/api/pandas.Index.fillna
 generated/pandas.Index.format,../reference/api/pandas.Index.format
-generated/pandas.Index.get_duplicates,../reference/api/pandas.Index.get_duplicates
 generated/pandas.Index.get_indexer_for,../reference/api/pandas.Index.get_indexer_for
 generated/pandas.Index.get_indexer,../reference/api/pandas.Index.get_indexer
 generated/pandas.Index.get_indexer_non_unique,../reference/api/pandas.Index.get_indexer_non_unique
@@ -1120,7 +1119,6 @@ generated/pandas.Series.pow,../reference/api/pandas.Series.pow
 generated/pandas.Series.prod,../reference/api/pandas.Series.prod
 generated/pandas.Series.product,../reference/api/pandas.Series.product
 generated/pandas.Series.ptp,../reference/api/pandas.Series.ptp
-generated/pandas.Series.put,../reference/api/pandas.Series.put
 generated/pandas.Series.quantile,../reference/api/pandas.Series.quantile
 generated/pandas.Series.radd,../reference/api/pandas.Series.radd
 generated/pandas.Series.rank,../reference/api/pandas.Series.rank

doc/source/reference/indexing.rst

@@ -152,7 +152,6 @@ Selecting
    Index.asof
    Index.asof_locs
    Index.contains
-   Index.get_duplicates
    Index.get_indexer
    Index.get_indexer_for
    Index.get_indexer_non_unique

doc/source/reference/series.rst

@@ -39,7 +39,6 @@ Attributes
    Series.empty
    Series.dtypes
    Series.name
-   Series.put
 
 Conversion
 ----------

doc/source/whatsnew/v1.0.0.rst

@@ -506,6 +506,9 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - Removed the previously deprecated :meth:`Series.get_value`, :meth:`Series.set_value`, :meth:`DataFrame.get_value`, :meth:`DataFrame.set_value` (:issue:`17739`)
 - Changed the the default value of `inplace` in :meth:`DataFrame.set_index` and :meth:`Series.set_axis`. It now defaults to ``False`` (:issue:`27600`)
 - Removed the previously deprecated :attr:`Series.cat.categorical`, :attr:`Series.cat.index`, :attr:`Series.cat.name` (:issue:`24751`)
+- Removed the previously deprecated ``time_rule`` keyword from (non-public) :func:`offsets.generate_range`, which has been moved to :func:`core.arrays._ranges.generate_range` (:issue:`24157`)
+- :meth:`DataFrame.loc` or :meth:`Series.loc` with listlike indexers and missing labels will no longer reindex (:issue:`17295`)
+- :meth:`DataFrame.to_excel` and :meth:`Series.to_excel` with non-existent columns will no longer reindex (:issue:`17295`)
 - Removed the previously deprecated "by" keyword from :meth:`DataFrame.sort_index`, use :meth:`DataFrame.sort_values` instead (:issue:`10726`)
 - Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`DataFrameGroupBy.aggregate`, :meth:`SeriesGroupBy.aggregate`, :meth:`Rolling.aggregate` (:issue:`18529`)
 - Passing ``datetime64`` data to :class:`TimedeltaIndex` or ``timedelta64`` data to ``DatetimeIndex`` now raises ``TypeError`` (:issue:`23539`, :issue:`23937`)
@@ -548,7 +551,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - Removed the previously properties :attr:`DataFrame.is_copy`, :attr:`Series.is_copy` (:issue:`18812`)
 - Removed the previously deprecated :meth:`DataFrame.get_ftype_counts`, :meth:`Series.get_ftype_counts` (:issue:`18243`)
 - Removed the previously deprecated :meth:`DataFrame.ftypes`, :meth:`Series.ftypes`, :meth:`Series.ftype` (:issue:`26744`)
-- Removed the previously deprecated :meth:`Index.get_duplicated`, use ``idx[idx.duplicated()].unique()`` instead (:issue:`20239`)
+- Removed the previously deprecated :meth:`Index.get_duplicates`, use ``idx[idx.duplicated()].unique()`` instead (:issue:`20239`)
 - Removed the previously deprecated :meth:`Series.clip_upper`, :meth:`Series.clip_lower`, :meth:`DataFrame.clip_upper`, :meth:`DataFrame.clip_lower` (:issue:`24203`)
 - Removed the ability to alter :attr:`DatetimeIndex.freq`, :attr:`TimedeltaIndex.freq`, or :attr:`PeriodIndex.freq` (:issue:`20772`)
 - Removed the previously deprecated :attr:`DatetimeIndex.offset` (:issue:`20730`)
@@ -562,6 +565,9 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - Passing multiple axes to :meth:`DataFrame.dropna` is no longer supported (:issue:`20995`)
 - Removed previously deprecated :meth:`Series.nonzero`, use `to_numpy().nonzero()` instead (:issue:`24048`)
 - Passing floating dtype ``codes`` to :meth:`Categorical.from_codes` is no longer supported, pass ``codes.astype(np.int64)`` instead (:issue:`21775`)
+- :meth:`Series.str.partition` and :meth:`Series.str.rpartition` no longer accept "pat" keyword, use "sep" instead (:issue:`23767`)
+- Removed the previously deprecated :meth:`Series.put` (:issue:`27106`)
+- Removed the previously deprecated :attr:`Series.real`, :attr:`Series.imag` (:issue:`27106`)
 - Removed the previously deprecated :meth:`Series.to_dense`, :meth:`DataFrame.to_dense` (:issue:`26684`)
 - Removed the previously deprecated :meth:`Index.dtype_str`, use ``str(index.dtype)`` instead (:issue:`27106`)
 - :meth:`Categorical.ravel` returns a :class:`Categorical` instead of a ``ndarray`` (:issue:`27199`)
@@ -658,6 +664,7 @@ Numeric
 - Bug in :class:`UInt64Index` precision loss while constructing from a list with values in the ``np.uint64`` range (:issue:`29526`)
 - Bug in :class:`NumericIndex` construction that caused indexing to fail when integers in the ``np.uint64`` range were used (:issue:`28023`)
 - Bug in :class:`NumericIndex` construction that caused :class:`UInt64Index` to be casted to :class:`Float64Index` when integers in the ``np.uint64`` range were used to index a :class:`DataFrame` (:issue:`28279`)
+- Bug in :meth:`Series.interpolate` when using method=`index` with an unsorted index, would previously return incorrect results. (:issue:`21037`)
 
 Conversion
 ^^^^^^^^^^

doc/sphinxext/announce.py

@@ -113,13 +113,13 @@ def build_string(revision_range, heading="Contributors"):
     components["authors"] = "* " + "\n* ".join(components["authors"])
 
     tpl = textwrap.dedent(
-        """\
-    {heading}
-    {uline}
+        f"""\
+    {components['heading']}
+    {components['uline']}
 
-    {author_message}
-    {authors}"""
-    ).format(**components)
+    {components['author_message']}
+    {components['authors']}"""
+    )
     return tpl
 
 

doc/sphinxext/contributors.py

@@ -27,7 +27,7 @@ def run(self):
         except git.GitCommandError as exc:
             return [
                 self.state.document.reporter.warning(
-                    "Cannot find contributors for range '{}': {}".format(range_, exc),
+                    f"Cannot find contributors for range {repr(range_)}: {exc}",
                     line=self.lineno,
                 )
             ]

pandas/_libs/parsers.pyx

@@ -63,11 +63,6 @@ from pandas.errors import (ParserError, DtypeWarning,
 
 lzma = _import_lzma()
 
-# Import CParserError as alias of ParserError for backwards compatibility.
-# Ultimately, we want to remove this import. See gh-12665 and gh-14479.
-CParserError = ParserError
-
-
 cdef:
     float64_t INF = <float64_t>np.inf
     float64_t NEGINF = -INF

pandas/compat/chainmap.py

@@ -10,6 +10,12 @@ def __setitem__(self, key, value):
         self.maps[0][key] = value
 
     def __delitem__(self, key):
+        """
+        Raises
+        ------
+        KeyError
+            If `key` doesn't exist.
+        """
         for mapping in self.maps:
             if key in mapping:
                 del mapping[key]

pandas/core/arrays/numpy_.py

@@ -1,4 +1,5 @@
 import numbers
+from typing import Union
 
 import numpy as np
 from numpy.lib.mixins import NDArrayOperatorsMixin
@@ -117,11 +118,12 @@ class PandasArray(ExtensionArray, ExtensionOpsMixin, NDArrayOperatorsMixin):
     # pandas internals, which turns off things like block consolidation.
     _typ = "npy_extension"
     __array_priority__ = 1000
+    _ndarray: np.ndarray
 
     # ------------------------------------------------------------------------
     # Constructors
 
-    def __init__(self, values, copy=False):
+    def __init__(self, values: Union[np.ndarray, "PandasArray"], copy: bool = False):
         if isinstance(values, type(self)):
             values = values._ndarray
         if not isinstance(values, np.ndarray):

pandas/core/base.py

@@ -114,9 +114,7 @@ def __setattr__(self, key, value):
             or key in type(self).__dict__
             or getattr(self, key, None) is not None
         ):
-            raise AttributeError(
-                "You cannot add any new attribute '{key}'".format(key=key)
-            )
+            raise AttributeError(f"You cannot add any new attribute '{key}'")
         object.__setattr__(self, key, value)
 
 
@@ -220,28 +218,22 @@ def _obj_with_exclusions(self):
 
     def __getitem__(self, key):
         if self._selection is not None:
-            raise IndexError(
-                "Column(s) {selection} already selected".format(
-                    selection=self._selection
-                )
-            )
+            raise IndexError(f"Column(s) {self._selection} already selected")
 
         if isinstance(key, (list, tuple, ABCSeries, ABCIndexClass, np.ndarray)):
             if len(self.obj.columns.intersection(key)) != len(key):
                 bad_keys = list(set(key).difference(self.obj.columns))
-                raise KeyError(
-                    "Columns not found: {missing}".format(missing=str(bad_keys)[1:-1])
-                )
+                raise KeyError(f"Columns not found: {str(bad_keys)[1:-1]}")
             return self._gotitem(list(key), ndim=2)
 
         elif not getattr(self, "as_index", False):
             if key not in self.obj.columns:
-                raise KeyError("Column not found: {key}".format(key=key))
+                raise KeyError(f"Column not found: {key}")
             return self._gotitem(key, ndim=2)
 
         else:
             if key not in self.obj:
-                raise KeyError("Column not found: {key}".format(key=key))
+                raise KeyError(f"Column not found: {key}")
             return self._gotitem(key, ndim=1)
 
     def _gotitem(self, key, ndim, subset=None):
@@ -293,8 +285,7 @@ def _try_aggregate_string_function(self, arg: str, *args, **kwargs):
                 return f(self, *args, **kwargs)
 
         raise AttributeError(
-            "'{arg}' is not a valid function for "
-            "'{cls}' object".format(arg=arg, cls=type(self).__name__)
+            f"'{arg}' is not a valid function for '{type(self).__name__}' object"
         )
 
     def _aggregate(self, arg, *args, **kwargs):
@@ -359,7 +350,7 @@ def _aggregate(self, arg, *args, **kwargs):
                     elif isinstance(obj, ABCSeries):
                         raise SpecificationError("nested renamer is not supported")
                     elif isinstance(obj, ABCDataFrame) and k not in obj.columns:
-                        raise KeyError("Column '{col}' does not exist!".format(col=k))
+                        raise KeyError(f"Column '{k}' does not exist!")
 
                 arg = new_arg
 
@@ -1101,9 +1092,7 @@ def _reduce(
         func = getattr(self, name, None)
         if func is None:
             raise TypeError(
-                "{klass} cannot perform the operation {op}".format(
-                    klass=type(self).__name__, op=name
-                )
+                f"{type(self).__name__} cannot perform the operation {name}"
             )
         return func(skipna=skipna, **kwds)