Merge remote-tracking branch 'upstream/master' into bisect

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master
@@ -132,15 +133,15 @@ jobs:
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
         echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
-      if: github.event_name == 'push'
+      if: ${{github.event_name == 'push' && github.ref == 'refs/head/master'}}
 
     - name: Upload web
       run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' --exclude='Pandas_Cheat_Sheet*' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
-      if: github.event_name == 'push'
+      if: ${{github.event_name == 'push' && github.ref == 'refs/head/master'}}
 
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
-      if: github.event_name == 'push'
+      if: ${{github.event_name == 'push' && github.ref == 'refs/head/master'}}
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs

.github/workflows/posix.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master

.github/workflows/pre-commit.yml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
 
 jobs:
   pre-commit:

.github/workflows/sdist.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - 1.3.x
   pull_request:
     branches:
       - master

asv_bench/benchmarks/algos/isin.py

@@ -1,10 +1,5 @@
 import numpy as np
 
-try:
-    from pandas.compat import np_version_under1p20
-except ImportError:
-    from pandas.compat.numpy import _np_version_under1p20 as np_version_under1p20
-
 from pandas import (
     Categorical,
     NaT,
@@ -283,10 +278,6 @@ class IsInLongSeriesLookUpDominates:
     def setup(self, dtype, MaxNumber, series_type):
         N = 10 ** 7
 
-        # https://github.com/pandas-dev/pandas/issues/39844
-        if not np_version_under1p20 and dtype in ("Int64", "Float64"):
-            raise NotImplementedError
-
         if series_type == "random_hits":
             array = np.random.randint(0, MaxNumber, N)
         if series_type == "random_misses":
@@ -297,7 +288,8 @@ def setup(self, dtype, MaxNumber, series_type):
             array = np.arange(N) + MaxNumber
 
         self.series = Series(array).astype(dtype)
-        self.values = np.arange(MaxNumber).astype(dtype)
+
+        self.values = np.arange(MaxNumber).astype(dtype.lower())
 
     def time_isin(self, dtypes, MaxNumber, series_type):
         self.series.isin(self.values)
@@ -313,16 +305,12 @@ class IsInLongSeriesValuesDominate:
     def setup(self, dtype, series_type):
         N = 10 ** 7
 
-        # https://github.com/pandas-dev/pandas/issues/39844
-        if not np_version_under1p20 and dtype in ("Int64", "Float64"):
-            raise NotImplementedError
-
         if series_type == "random":
             vals = np.random.randint(0, 10 * N, N)
         if series_type == "monotone":
             vals = np.arange(N)
 
-        self.values = vals.astype(dtype)
+        self.values = vals.astype(dtype.lower())
         M = 10 ** 6 + 1
         self.series = Series(np.arange(M)).astype(dtype)
 

doc/source/user_guide/categorical.rst

@@ -777,8 +777,8 @@ value is included in the ``categories``:
     df
     try:
         df.iloc[2:4, :] = [["c", 3], ["c", 3]]
-    except ValueError as e:
-        print("ValueError:", str(e))
+    except TypeError as e:
+        print("TypeError:", str(e))
 
 Setting values by assigning categorical data will also check that the ``categories`` match:
 
@@ -788,8 +788,8 @@ Setting values by assigning categorical data will also check that the ``categori
     df
     try:
         df.loc["j":"k", "cats"] = pd.Categorical(["b", "b"], categories=["a", "b", "c"])
-    except ValueError as e:
-        print("ValueError:", str(e))
+    except TypeError as e:
+        print("TypeError:", str(e))
 
 Assigning a ``Categorical`` to parts of a column of other types will use the values:
 

doc/source/user_guide/groupby.rst

@@ -391,7 +391,6 @@ something different for each of the columns. Thus, using ``[]`` similar to
 getting a column from a DataFrame, you can do:
 
 .. ipython:: python
-   :suppress:
 
    df = pd.DataFrame(
        {
@@ -402,7 +401,7 @@ getting a column from a DataFrame, you can do:
        }
    )
 
-.. ipython:: python
+   df
 
    grouped = df.groupby(["A"])
    grouped_C = grouped["C"]

doc/source/user_guide/style.ipynb

@@ -1201,7 +1201,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df2.style.bar(align=0, vmin=-2.5, vmax=2.5, color=['#d65f5f', '#5fba7d'],\n",
+    "df2.style.bar(align=0, vmin=-2.5, vmax=2.5, color=['#d65f5f', '#5fba7d'], height=50,\n",
     "              width=60, props=\"width: 120px; border-right: 1px solid black;\").format('{:.3f}', na_rep=\"\")"
    ]
   },

doc/source/whatsnew/v1.3.1.rst

@@ -22,7 +22,8 @@ Fixed regressions
 - Performance regression in :class:`DataFrame` in reduction operations requiring casting such as :meth:`DataFrame.mean` on integer data (:issue:`38592`)
 - Performance regression in :meth:`DataFrame.to_dict` and :meth:`Series.to_dict` when ``orient`` argument one of "records", "dict", or "split" (:issue:`42352`)
 - Fixed regression in indexing with a ``list`` subclass incorrectly raising ``TypeError`` (:issue:`42433`, :issue:`42461`)
--
+- Fixed regression in :meth:`DataFrame.isin` and :meth:`Series.isin` raising ``TypeError`` with nullable data containing at least one missing value (:issue:`42405`)
+- Regression in :func:`concat` between objects with bool dtype and integer dtype casting to object instead of to integer (:issue:`42092`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.4.0.rst

@@ -154,7 +154,7 @@ Deprecations
 - Deprecated ``method`` argument in :meth:`Index.get_loc`, use ``index.get_indexer([label], method=...)`` instead (:issue:`42269`)
 - Deprecated treating integer keys in :meth:`Series.__setitem__` as positional when the index is a :class:`Float64Index` not containing the key, a :class:`IntervalIndex` with no entries containing the key, or a :class:`MultiIndex` with leading :class:`Float64Index` level not containing the key (:issue:`33469`)
 - Deprecated treating ``numpy.datetime64`` objects as UTC times when passed to the :class:`Timestamp` constructor along with a timezone. In a future version, these will be treated as wall-times. To retain the old behavior, use ``Timestamp(dt64).tz_localize("UTC").tz_convert(tz)`` (:issue:`24559`)
--
+- Deprecated ignoring missing labels when indexing with a sequence of labels on a level of a MultiIndex (:issue:`42351`)
 
 .. ---------------------------------------------------------------------------
 
@@ -174,12 +174,17 @@ Bug fixes
 
 Categorical
 ^^^^^^^^^^^
--
+- Bug in setting dtype-incompatible values into a :class:`Categorical` (or ``Series`` or ``DataFrame`` backed by ``Categorical``) raising ``ValueError`` instead of ``TypeError`` (:issue:`41919`)
+- Bug in :meth:`Categorical.searchsorted` when passing a dtype-incompatible value raising ``KeyError`` instead of ``TypeError`` (:issue:`41919`)
+- Bug in :meth:`Series.where` with ``CategoricalDtype`` when passing a dtype-incompatible value raising ``ValueError`` instead of ``TypeError`` (:issue:`41919`)
+- Bug in :meth:`Categorical.fillna` when passing a dtype-incompatible value raising ``ValueError`` instead of ``TypeError`` (:issue:`41919`)
+- Bug in :meth:`Categorical.fillna` with a tuple-like category raising ``ValueError`` instead of ``TypeError`` when filling with a non-category tuple (:issue:`41919`)
 -
 
 Datetimelike
 ^^^^^^^^^^^^
 - Bug in :func:`to_datetime` returning pd.NaT for inputs that produce duplicated values, when ``cache=True`` (:issue:`42259`)
+- Bug in :class:`DataFrame` constructor unnecessarily copying non-datetimelike 2D object arrays (:issue:`39272`)
 -
 
 Timedelta
@@ -215,8 +220,11 @@ Interval
 
 Indexing
 ^^^^^^^^
-- Bug in indexing on a :class:`Series` or :class:`DataFrame` with a :class:`DatetimeIndex` when passing a string, the return type depended on whether the index was monotonic (:issue:`24892`)
 - Bug in :meth:`DataFrame.truncate` and :meth:`Series.truncate` when the object's Index has a length greater than one but only one unique value (:issue:`42365`)
+- Bug in :meth:`Series.loc` when with a :class:`MultiIndex` whose first level contains only ``np.nan`` values (:issue:`42055`)
+- Bug in indexing on a :class:`Series` or :class:`DataFrame` with a :class:`DatetimeIndex` when passing a string, the return type depended on whether the index was monotonic (:issue:`24892`)
+- Bug in indexing on a :class:`MultiIndex` failing to drop scalar levels when the indexer is a tuple containing a datetime-like string (:issue:`42476`)
+-
 
 Missing
 ^^^^^^^
@@ -225,13 +233,15 @@ Missing
 
 MultiIndex
 ^^^^^^^^^^
+- Bug in :meth:`MultiIndex.get_loc` where the first level is a :class:`DatetimeIndex` and a string key is passed (:issue:`42465`)
 - Bug in :meth:`MultiIndex.reindex` when passing a ``level`` that corresponds to an ``ExtensionDtype`` level (:issue:`42043`)
 -
 
 I/O
 ^^^
 - Bug in :func:`read_excel` attempting to read chart sheets from .xlsx files (:issue:`41448`)
--
+- Bug in :func:`json_normalize` where ``errors=ignore`` could fail to ignore missing values of ``meta`` when ``record_path`` has a length greater than one (:issue:`41876`)
+- Bug in :func:`read_csv` with multi-header input and arguments referencing column names as tuples (:issue:`42446`)
 -
 
 Period

pandas/_libs/internals.pyi

@@ -1,6 +1,7 @@
 from typing import (
     Iterator,
     Sequence,
+    final,
     overload,
 )
 
@@ -50,10 +51,12 @@ class SharedBlock:
 
 class NumpyBlock(SharedBlock):
     values: np.ndarray
+    @final
     def getitem_block_index(self: T, slicer: slice) -> T: ...
 
 class NDArrayBackedBlock(SharedBlock):
     values: NDArrayBackedExtensionArray
+    @final
     def getitem_block_index(self: T, slicer: slice) -> T: ...
 
 class Block(SharedBlock): ...

pandas/_libs/internals.pyx

@@ -517,7 +517,6 @@ cdef class NumpyBlock(SharedBlock):
         #  set placement and ndim
         self.values = values
 
-    # @final  # not useful in cython, but we _would_ annotate with @final
     cpdef NumpyBlock getitem_block_index(self, slice slicer):
         """
         Perform __getitem__-like specialized to slicing along index.
@@ -540,7 +539,6 @@ cdef class NDArrayBackedBlock(SharedBlock):
         #  set placement and ndim
         self.values = values
 
-    # @final  # not useful in cython, but we _would_ annotate with @final
     cpdef NDArrayBackedBlock getitem_block_index(self, slice slicer):
         """
         Perform __getitem__-like specialized to slicing along index.

pandas/_libs/parsers.pyx

@@ -1280,6 +1280,8 @@ cdef class TextReader:
                 # generate extra (bogus) headers if there are more columns than headers
                 if j >= len(self.header[0]):
                     return j
+                elif self.has_mi_columns:
+                    return tuple(header_row[j] for header_row in self.header)
                 else:
                     return self.header[0][j]
             else:

pandas/_typing.py

@@ -37,10 +37,7 @@
 # and use a string literal forward reference to it in subsequent types
 # https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles
 if TYPE_CHECKING:
-    from typing import (
-        TypedDict,
-        final,
-    )
+    from typing import TypedDict
 
     import numpy.typing as npt
 
@@ -76,8 +73,6 @@
     from pandas.tseries.offsets import DateOffset
 else:
     npt: Any = None
-    # typing.final does not exist until py38
-    final = lambda x: x
     # typing.TypedDict does not exist until py38
     TypedDict = dict
 

pandas/core/algorithms.py

@@ -531,7 +531,7 @@ def factorize_array(
     mask: np.ndarray | None = None,
 ) -> tuple[npt.NDArray[np.intp], np.ndarray]:
     """
-    Factorize an array-like to codes and uniques.
+    Factorize a numpy array to codes and uniques.
 
     This doesn't do any coercion of types or unboxing before factorization.
 
@@ -910,7 +910,7 @@ def duplicated(
 
     Parameters
     ----------
-    values : ndarray-like
+    values : nd.array, ExtensionArray or Series
         Array over which to check for duplicate values.
     keep : {'first', 'last', False}, default 'first'
         - ``first`` : Mark duplicates as ``True`` except for the first
@@ -1412,8 +1412,8 @@ def take(
 
     Parameters
     ----------
-    arr : sequence
-        Non array-likes (sequences without a dtype) are coerced
+    arr : array-like or scalar value
+        Non array-likes (sequences/scalars without a dtype) are coerced
         to an ndarray.
     indices : sequence of integers
         Indices to be taken.
@@ -1523,11 +1523,11 @@ def searchsorted(arr, value, side="left", sorter=None) -> np.ndarray:
 
     Parameters
     ----------
-    arr: array-like
+    arr: np.ndarray, ExtensionArray, Series
         Input array. If `sorter` is None, then it must be sorted in
         ascending order, otherwise `sorter` must be an array of indices
         that sort it.
-    value : array-like
+    value : array-like or scalar
         Values to insert into `arr`.
     side : {'left', 'right'}, optional
         If 'left', the index of the first suitable location found is given.

pandas/core/arrays/base.py

@@ -826,8 +826,8 @@ def searchsorted(self, value, side="left", sorter=None):
 
         Parameters
         ----------
-        value : array-like
-            Values to insert into `self`.
+        value : array-like, list or scalar
+            Value(s) to insert into `self`.
         side : {'left', 'right'}, optional
             If 'left', the index of the first suitable location found is given.
             If 'right', return the last such index.  If there is no suitable