Merge branch '1.2.x' of https://github.com/pandas-dev/pandas into 1.2.x

Author: rhshadrach

asv_bench/benchmarks/series_methods.py

@@ -87,8 +87,8 @@ def setup(self):
         self.vals_short = np.arange(2).astype(object)
         self.vals_long = np.arange(10 ** 5).astype(object)
         # because of nans floats are special:
-        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float)).astype(object)
-        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float).astype(object)
+        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float_)).astype(object)
+        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float_).astype(object)
 
     def time_isin_nans(self):
         # if nan-objects are different objects,

doc/source/conf.py

@@ -409,7 +409,7 @@
 if pattern is None:
     intersphinx_mapping = {
         "dateutil": ("https://dateutil.readthedocs.io/en/latest/", None),
-        "matplotlib": ("https://matplotlib.org/", None),
+        "matplotlib": ("https://matplotlib.org/stable/", None),
         "numpy": ("https://numpy.org/doc/stable/", None),
         "pandas-gbq": ("https://pandas-gbq.readthedocs.io/en/latest/", None),
         "py": ("https://pylib.readthedocs.io/en/latest/", None),

doc/source/user_guide/cookbook.rst

@@ -1410,7 +1410,7 @@ Often it's useful to obtain the lower (or upper) triangular form of a correlatio
 
     corr_mat.where(mask)
 
-The ``method`` argument within ``DataFrame.corr`` can accept a callable in addition to the named correlation types.  Here we compute the ``distance correlation <https://en.wikipedia.org/wiki/Distance_correlation>``__ matrix for a ``DataFrame`` object.
+The ``method`` argument within ``DataFrame.corr`` can accept a callable in addition to the named correlation types.  Here we compute the `distance correlation <https://en.wikipedia.org/wiki/Distance_correlation>`__ matrix for a ``DataFrame`` object.
 
 .. ipython:: python
 

doc/source/whatsnew/index.rst

@@ -16,6 +16,7 @@ Version 1.2
 .. toctree::
    :maxdepth: 2
 
+   v1.2.3
    v1.2.2
    v1.2.1
    v1.2.0

doc/source/whatsnew/v1.2.2.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_122:
 
-What's new in 1.2.2 (February ??, 2021)
+What's new in 1.2.2 (February 09, 2021)
 ---------------------------------------
 
 These are the changes in pandas 1.2.2. See :ref:`release` for a full changelog
@@ -17,10 +17,16 @@ Fixed regressions
 
 - Fixed regression in :func:`read_excel` that caused it to raise ``AttributeError`` when checking version of older xlrd versions (:issue:`38955`)
 - Fixed regression in :class:`DataFrame` constructor reordering element when construction from datetime ndarray with dtype not ``"datetime64[ns]"`` (:issue:`39422`)
+- Fixed regression in :meth:`DataFrame.astype` and :meth:`Series.astype` not casting to bytes dtype (:issue:`39474`)
 - Fixed regression in :meth:`~DataFrame.to_pickle` failing to create bz2/xz compressed pickle files with ``protocol=5`` (:issue:`39002`)
 - Fixed regression in :func:`pandas.testing.assert_series_equal` and :func:`pandas.testing.assert_frame_equal` always raising ``AssertionError`` when comparing extension dtypes (:issue:`39410`)
 - Fixed regression in :meth:`~DataFrame.to_csv` opening ``codecs.StreamWriter`` in binary mode instead of in text mode and ignoring user-provided ``mode`` (:issue:`39247`)
--
+- Fixed regression in :meth:`Categorical.astype` casting to incorrect dtype when ``np.int32`` is passed to dtype argument (:issue:`39402`)
+- Fixed regression in :meth:`~DataFrame.to_excel` creating corrupt files when appending (``mode="a"``) to an existing file (:issue:`39576`)
+- Fixed regression in :meth:`DataFrame.transform` failing in case of an empty DataFrame or Series (:issue:`39636`)
+- Fixed regression in :meth:`~DataFrame.groupby` or :meth:`~DataFrame.resample` when aggregating an all-NaN or numeric object dtype column (:issue:`39329`)
+- Fixed regression in :meth:`.Rolling.count` where the ``min_periods`` argument would be set to ``0`` after the operation (:issue:`39554`)
+- Fixed regression in :func:`read_excel` that incorrectly raised when the argument ``io`` was a non-path and non-buffer and the ``engine`` argument was specified (:issue:`39528`)
 
 .. ---------------------------------------------------------------------------
 
@@ -30,17 +36,8 @@ Bug fixes
 ~~~~~~~~~
 
 - :func:`pandas.read_excel` error message when a specified ``sheetname`` does not exist is now uniform across engines (:issue:`39250`)
--
-
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_122.other:
-
-Other
-~~~~~
-
--
--
+- Fixed bug in :func:`pandas.read_excel` producing incorrect results when the engine ``openpyxl`` is used and the excel file is missing or has incorrect dimension information; the fix requires ``openpyxl`` >= 3.0.0, prior versions may still fail (:issue:`38956`, :issue:`39001`)
+- Fixed bug in :func:`pandas.read_excel` sometimes producing a ``DataFrame`` with trailing rows of ``np.nan`` when the engine ``openpyxl`` is used (:issue:`39181`)
 
 .. ---------------------------------------------------------------------------
 
@@ -49,4 +46,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.2.1..v1.2.2|HEAD
+.. contributors:: v1.2.1..v1.2.2

doc/source/whatsnew/v1.2.3.rst

@@ -0,0 +1,55 @@
+.. _whatsnew_123:
+
+What's new in 1.2.3 (March ??, 2021)
+------------------------------------
+
+These are the changes in pandas 1.2.3. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+
+- Fixed regression in :meth:`~DataFrame.to_excel` raising ``KeyError`` when giving duplicate columns with ``columns`` attribute (:issue:`39695`)
+- Fixed regression in nullable integer unary ops propagating mask on assignment (:issue:`39943`)
+- Fixed regression in :meth:`DataFrame.__setitem__` not aligning :class:`DataFrame` on right-hand side for boolean indexer (:issue:`39931`)
+- Fixed regression in :meth:`~DataFrame.to_json` failing to use ``compression`` with URL-like paths that are internally opened in binary mode or with user-provided file objects that are opened in binary mode (:issue:`39985`)
+- Fixed regression in :meth:`~Series.sort_index` and :meth:`~DataFrame.sort_index`,
+  which exited with an ungraceful error when having kwarg ``ascending=None`` passed (:issue:`39434`).
+  Passing ``ascending=None`` is still considered invalid,
+  and the new error message suggests a proper usage
+  (``ascending`` must be a boolean or a list-like boolean).
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.other:
+
+Other
+~~~~~
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_123.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.2.2..v1.2.3|HEAD

environment.yml

@@ -3,7 +3,7 @@ channels:
   - conda-forge
 dependencies:
   # required
-  - numpy>=1.16.5, <1.20 # gh-39513
+  - numpy>=1.16.5
   - python=3
   - python-dateutil>=2.7.3
   - pytz
@@ -113,5 +113,5 @@ dependencies:
   - tabulate>=0.8.3  # DataFrame.to_markdown
   - natsort  # DataFrame.sort_values
   - pip:
-    - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master
-    - git+https://github.com/numpy/numpydoc
+    - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@2488b7defbd3d753dd5fcfc890fc4a7e79d25103
+    - numpydoc < 1.2  # 2021-02-09 1.2dev breaking CI

pandas/compat/_optional.py

@@ -15,7 +15,7 @@
     "matplotlib": "2.2.3",
     "numexpr": "2.6.8",
     "odfpy": "1.3.0",
-    "openpyxl": "2.5.7",
+    "openpyxl": "2.6.0",
     "pandas_gbq": "0.12.0",
     "pyarrow": "0.15.0",
     "pytest": "5.0.1",

pandas/conftest.py

@@ -1188,6 +1188,32 @@ def any_nullable_int_dtype(request):
     return request.param
 
 
+@pytest.fixture(params=tm.ALL_INT_DTYPES + tm.ALL_EA_INT_DTYPES)
+def any_int_or_nullable_int_dtype(request):
+    """
+    Parameterized fixture for any nullable integer dtype.
+
+    * int
+    * 'int8'
+    * 'uint8'
+    * 'int16'
+    * 'uint16'
+    * 'int32'
+    * 'uint32'
+    * 'int64'
+    * 'uint64'
+    * 'UInt8'
+    * 'Int8'
+    * 'UInt16'
+    * 'Int16'
+    * 'UInt32'
+    * 'Int32'
+    * 'UInt64'
+    * 'Int64'
+    """
+    return request.param
+
+
 @pytest.fixture(params=tm.ALL_EA_INT_DTYPES + tm.FLOAT_EA_DTYPES)
 def any_numeric_dtype(request):
     """

pandas/core/aggregation.py

@@ -456,7 +456,7 @@ def transform(
 
     # Functions that transform may return empty Series/DataFrame
     # when the dtype is not appropriate
-    if isinstance(result, (ABCSeries, ABCDataFrame)) and result.empty:
+    if isinstance(result, (ABCSeries, ABCDataFrame)) and result.empty and not obj.empty:
         raise ValueError("Transform function failed")
     if not isinstance(result, (ABCSeries, ABCDataFrame)) or not result.index.equals(
         obj.index

pandas/core/arrays/categorical.py

@@ -427,16 +427,16 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
         else:
             # GH8628 (PERF): astype category codes instead of astyping array
             try:
-                astyped_cats = self.categories.astype(dtype=dtype, copy=copy)
+                new_cats = np.asarray(self.categories)
+                new_cats = new_cats.astype(dtype=dtype, copy=copy)
             except (
                 TypeError,  # downstream error msg for CategoricalIndex is misleading
                 ValueError,
             ):
                 msg = f"Cannot cast {self.categories.dtype} dtype to {dtype}"
                 raise ValueError(msg)
 
-            astyped_cats = extract_array(astyped_cats, extract_numpy=True)
-            result = take_1d(astyped_cats, libalgos.ensure_platform_int(self._codes))
+            result = take_1d(new_cats, libalgos.ensure_platform_int(self._codes))
 
         return result
 

pandas/core/arrays/integer.py

@@ -348,13 +348,13 @@ def __init__(self, values: np.ndarray, mask: np.ndarray, copy: bool = False):
         super().__init__(values, mask, copy=copy)
 
     def __neg__(self):
-        return type(self)(-self._data, self._mask)
+        return type(self)(-self._data, self._mask.copy())
 
     def __pos__(self):
         return self
 
     def __abs__(self):
-        return type(self)(np.abs(self._data), self._mask)
+        return type(self)(np.abs(self._data), self._mask.copy())
 
     @classmethod
     def _from_sequence(

pandas/core/arrays/masked.py

@@ -142,7 +142,7 @@ def __len__(self) -> int:
         return len(self._data)
 
     def __invert__(self: BaseMaskedArrayT) -> BaseMaskedArrayT:
-        return type(self)(~self._data, self._mask)
+        return type(self)(~self._data, self._mask.copy())
 
     def to_numpy(
         self, dtype=None, copy: bool = False, na_value: Scalar = lib.no_default

pandas/core/dtypes/cast.py

@@ -1137,7 +1137,7 @@ def soft_convert_objects(
         # bound of nanosecond-resolution 64-bit integers.
         try:
             values = lib.maybe_convert_objects(values, convert_datetime=True)
-        except OutOfBoundsDatetime:
+        except (OutOfBoundsDatetime, ValueError):
             pass
 
     if timedelta and is_object_dtype(values.dtype):

pandas/core/frame.py

@@ -3179,6 +3179,9 @@ def _setitem_array(self, key, value):
             key = check_bool_indexer(self.index, key)
             indexer = key.nonzero()[0]
             self._check_setitem_copy()
+            if isinstance(value, DataFrame):
+                # GH#39931 reindex since iloc does not align
+                value = value.reindex(self.index.take(indexer))
             self.iloc[indexer] = value
         else:
             if isinstance(value, DataFrame):
@@ -3951,8 +3954,8 @@ def lookup(self, row_labels, col_labels) -> np.ndarray:
         .. deprecated:: 1.2.0
             DataFrame.lookup is deprecated,
             use DataFrame.melt and DataFrame.loc instead.
-            For an example see :meth:`~pandas.DataFrame.lookup`
-            in the user guide.
+            For further details see
+            :ref:`Looking up values by index/column labels <indexing.lookup>`.
 
         Parameters
         ----------
@@ -5479,7 +5482,7 @@ def sort_index(
         self,
         axis=0,
         level=None,
-        ascending: bool = True,
+        ascending: Union[Union[bool, int], Sequence[Union[bool, int]]] = True,
         inplace: bool = False,
         kind: str = "quicksort",
         na_position: str = "last",
@@ -5500,7 +5503,7 @@ def sort_index(
             and 1 identifies the columns.
         level : int or level name or list of ints or list of level names
             If not None, sort on values in specified index level(s).
-        ascending : bool or list of bools, default True
+        ascending : bool or list-like of bools, default True
             Sort ascending vs. descending. When the index is a MultiIndex the
             sort direction can be controlled for each level individually.
         inplace : bool, default False

pandas/core/generic.py

@@ -56,6 +56,7 @@
 from pandas.errors import AbstractMethodError, InvalidIndexError
 from pandas.util._decorators import doc, rewrite_axis_style_signature
 from pandas.util._validators import (
+    validate_ascending,
     validate_bool_kwarg,
     validate_fillna_kwargs,
     validate_percentile,
@@ -4518,7 +4519,7 @@ def sort_index(
         self,
         axis=0,
         level=None,
-        ascending: bool_t = True,
+        ascending: Union[Union[bool_t, int], Sequence[Union[bool_t, int]]] = True,
         inplace: bool_t = False,
         kind: str = "quicksort",
         na_position: str = "last",
@@ -4529,6 +4530,8 @@ def sort_index(
 
         inplace = validate_bool_kwarg(inplace, "inplace")
         axis = self._get_axis_number(axis)
+        ascending = validate_ascending(ascending)
+
         target = self._get_axis(axis)
 
         indexer = get_indexer_indexer(
@@ -6438,6 +6441,7 @@ def fillna(
             return result.__finalize__(self, method="fillna")
 
     @final
+    @doc(klass=_shared_doc_kwargs["klass"])
     def ffill(
         self: FrameOrSeries,
         axis=None,
@@ -6460,6 +6464,7 @@ def ffill(
     pad = ffill
 
     @final
+    @doc(klass=_shared_doc_kwargs["klass"])
     def bfill(
         self: FrameOrSeries,
         axis=None,

pandas/core/groupby/generic.py

@@ -1078,11 +1078,18 @@ def py_fallback(bvalues: ArrayLike) -> ArrayLike:
             #  in the operation.  We un-split here.
             result = result._consolidate()
             assert isinstance(result, (Series, DataFrame))  # for mypy
-            assert len(result._mgr.blocks) == 1
+            mgr = result._mgr
+            assert isinstance(mgr, BlockManager)
 
             # unwrap DataFrame to get array
-            result = result._mgr.blocks[0].values
-            return result
+            if len(mgr.blocks) != 1:
+                # We've split an object block! Everything we've assumed
+                # about a single block input returning a single block output
+                # is a lie. See eg GH-39329
+                return mgr.as_array()
+            else:
+                result = mgr.blocks[0].values
+                return result
 
         def blk_func(bvalues: ArrayLike) -> ArrayLike:
 

pandas/core/indexes/multi.py

@@ -1286,16 +1286,18 @@ def _format_native_types(self, na_rep="nan", **kwargs):
 
         # go through the levels and format them
         for level, level_codes in zip(self.levels, self.codes):
-            level = level._format_native_types(na_rep=na_rep, **kwargs)
+            level_strs = level._format_native_types(na_rep=na_rep, **kwargs)
             # add nan values, if there are any
             mask = level_codes == -1
             if mask.any():
-                nan_index = len(level)
-                level = np.append(level, na_rep)
+                nan_index = len(level_strs)
+                # numpy 1.21 deprecated implicit string casting
+                level_strs = level_strs.astype(str)
+                level_strs = np.append(level_strs, na_rep)
                 assert not level_codes.flags.writeable  # i.e. copy is needed
                 level_codes = level_codes.copy()  # make writeable
                 level_codes[mask] = nan_index
-            new_levels.append(level)
+            new_levels.append(level_strs)
             new_codes.append(level_codes)
 
         if len(new_levels) == 1: