Merge remote-tracking branch 'upstream/main' into cow-warning-chained

Author: jorisvandenbossche

.github/workflows/unit-tests.yml

@@ -240,7 +240,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -278,7 +278,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -351,7 +351,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps
           python -m pip list
 

.github/workflows/wheels.yml

@@ -181,7 +181,7 @@ jobs:
         shell: pwsh
         run: |
           $TST_CMD = @"
-          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17;
+          python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
           python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
           "@

.pre-commit-config.yaml

@@ -20,11 +20,11 @@ ci:
 repos:
 -   repo: https://github.com/hauntsaninja/black-pre-commit-mirror
     # black compiled with mypyc
-    rev: 23.9.1
+    rev: 23.10.1
     hooks:
       - id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.291
+    rev: v0.1.4
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -34,14 +34,14 @@ repos:
         alias: ruff-selected-autofixes
         args: [--select, "ANN001,ANN204", --fix-only, --exit-non-zero-on-fix]
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.9.1'
+    rev: 'v2.10'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
         pass_filenames: true
         require_serial: false
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown, cython, c]
@@ -52,7 +52,7 @@ repos:
     -   id: cython-lint
     -   id: double-quote-cython-strings
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: check-ast
     -   id: check-case-conflict
@@ -71,7 +71,7 @@ repos:
         args: [--remove]
     -   id: trailing-whitespace
 -   repo: https://github.com/pylint-dev/pylint
-    rev: v3.0.0b0
+    rev: v3.0.1
     hooks:
     -   id: pylint
         stages: [manual]
@@ -94,7 +94,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.13.0
+    rev: v3.15.0
     hooks:
     -   id: pyupgrade
         args: [--py39-plus]
@@ -111,11 +111,11 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.6.8
+    rev: v0.8.1
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: ea59a72
+    rev: v17.0.4
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include

ci/deps/actions-310.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-311-downstream_compat.yaml

@@ -15,7 +15,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-311-numpydev.yaml

@@ -18,7 +18,6 @@ dependencies:
   # causes an InternalError within pytest
   - pytest-xdist>=2.2.0, <3
   - hypothesis>=6.46.1
-  - pytest-asyncio>=0.17.0
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-311-pyarrownightly.yaml

@@ -15,7 +15,6 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
-  - pytest-asyncio>=0.17.0
 
   # required dependencies
   - python-dateutil

ci/deps/actions-311.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-39-minimum_versions.yaml

@@ -16,7 +16,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-39.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/deps/actions-pypy-39.yaml

@@ -16,7 +16,6 @@ dependencies:
   # test dependencies
   - pytest>=7.3.2
   - pytest-cov
-  - pytest-asyncio>=0.17.0
   - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
 

ci/deps/circle-310-arm64.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - pytest-localserver>=0.7.1
   - boto3
 

ci/meta.yaml

@@ -64,7 +64,6 @@ test:
   requires:
     - pip
     - pytest >=7.3.2
-    - pytest-asyncio >=0.17.0
     - pytest-xdist >=2.2.0
     - pytest-cov
     - hypothesis >=6.46.1

doc/source/development/debugging_extensions.rst

@@ -23,7 +23,7 @@ By default building pandas from source will generate a release build. To generat
 
 .. note::
 
-   conda environements update CFLAGS/CPPFLAGS with flags that are geared towards generating releases. If using conda, you may need to set ``CFLAGS="$CFLAGS -O0"`` and ``CPPFLAGS="$CPPFLAGS -O0"`` to ensure optimizations are turned off for debugging
+   conda environments update CFLAGS/CPPFLAGS with flags that are geared towards generating releases. If using conda, you may need to set ``CFLAGS="$CFLAGS -O0"`` and ``CPPFLAGS="$CPPFLAGS -O0"`` to ensure optimizations are turned off for debugging
 
 By specifying ``builddir="debug"`` all of the targets will be built and placed in the debug directory relative to the project root. This helps to keep your debug and release artifacts separate; you are of course able to choose a different directory name or omit altogether if you do not care to separate build types.
 

doc/source/development/extending.rst

@@ -99,7 +99,7 @@ The interface consists of two classes.
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 A :class:`pandas.api.extensions.ExtensionDtype` is similar to a ``numpy.dtype`` object. It describes the
-data type. Implementors are responsible for a few unique items like the name.
+data type. Implementers are responsible for a few unique items like the name.
 
 One particularly important item is the ``type`` property. This should be the
 class that is the scalar type for your data. For example, if you were writing an

doc/source/getting_started/tutorials.rst

@@ -115,7 +115,6 @@ Various tutorials
 * `Statistical Data Analysis in Python, tutorial videos, by Christopher Fonnesbeck from SciPy 2013 <https://conference.scipy.org/scipy2013/tutorial_detail.php?id=109>`_
 * `Financial analysis in Python, by Thomas Wiecki <https://nbviewer.org/github/twiecki/financial-analysis-python-tutorial/blob/master/1.%20Pandas%20Basics.ipynb>`_
 * `Intro to pandas data structures, by Greg Reda <http://www.gregreda.com/2013/10/26/intro-to-pandas-data-structures/>`_
-* `Pandas and Python: Top 10, by Manish Amde <https://manishamde.github.io/blog/2013/03/07/pandas-and-python-top-10/>`_
 * `Pandas DataFrames Tutorial, by Karlijn Willems <https://www.datacamp.com/community/tutorials/pandas-tutorial-dataframe-python>`_
 * `A concise tutorial with real life examples <https://tutswiki.com/pandas-cookbook/chapter1/>`_
 * `430+ Searchable Pandas recipes by Isshin Inada <https://skytowner.com/explore/pandas_recipes_reference>`_

doc/source/user_guide/merging.rst

@@ -525,7 +525,7 @@ Performing an outer join with duplicate join keys in :class:`DataFrame`
 
 .. warning::
 
-  Merging on duplicate keys sigificantly increase the dimensions of the result
+  Merging on duplicate keys significantly increase the dimensions of the result
   and can cause a memory overflow.
 
 .. _merging.validation:

doc/source/user_guide/reshaping.rst

@@ -480,7 +480,7 @@ The values can be cast to a different type using the ``dtype`` argument.
 
 .. versionadded:: 1.5.0
 
-:func:`~pandas.from_dummies` coverts the output of :func:`~pandas.get_dummies` back into
+:func:`~pandas.from_dummies` converts the output of :func:`~pandas.get_dummies` back into
 a :class:`Series` of categorical values from indicator values.
 
 .. ipython:: python

doc/source/whatsnew/v1.0.0.rst

@@ -1079,7 +1079,7 @@ Datetimelike
 - Bug in masking datetime-like arrays with a boolean mask of an incorrect length not raising an ``IndexError`` (:issue:`30308`)
 - Bug in :attr:`Timestamp.resolution` being a property instead of a class attribute (:issue:`29910`)
 - Bug in :func:`pandas.to_datetime` when called with ``None`` raising ``TypeError`` instead of returning ``NaT`` (:issue:`30011`)
-- Bug in :func:`pandas.to_datetime` failing for ``deques`` when using ``cache=True`` (the default) (:issue:`29403`)
+- Bug in :func:`pandas.to_datetime` failing for ``dequeues`` when using ``cache=True`` (the default) (:issue:`29403`)
 - Bug in :meth:`Series.item` with ``datetime64`` or ``timedelta64`` dtype, :meth:`DatetimeIndex.item`, and :meth:`TimedeltaIndex.item` returning an integer instead of a :class:`Timestamp` or :class:`Timedelta` (:issue:`30175`)
 - Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`)
 - Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`)

doc/source/whatsnew/v2.2.0.rst

@@ -324,8 +324,11 @@ Performance improvements
 - Performance improvement in :meth:`DataFrame.groupby` when aggregating pyarrow timestamp and duration dtypes (:issue:`55031`)
 - Performance improvement in :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` when indexed by a :class:`MultiIndex` (:issue:`54835`)
 - Performance improvement in :meth:`Index.difference` (:issue:`55108`)
+- Performance improvement in :meth:`MultiIndex.get_indexer` when ``method`` is not ``None`` (:issue:`55839`)
 - Performance improvement in :meth:`Series.duplicated` for pyarrow dtypes (:issue:`55255`)
+- Performance improvement in :meth:`Series.str` methods (:issue:`55736`)
 - Performance improvement in :meth:`SeriesGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`DataFrameGroupBy.idxmin` (:issue:`54234`)
+- Performance improvement when indexing into a non-unique index (:issue:`55816`)
 - Performance improvement when indexing with more than 4 keys (:issue:`54550`)
 - Performance improvement when localizing time to UTC (:issue:`55241`)
 
@@ -346,6 +349,7 @@ Datetimelike
 - Bug in :class:`DatetimeIndex` construction when passing both a ``tz`` and either ``dayfirst`` or ``yearfirst`` ignoring dayfirst/yearfirst (:issue:`55813`)
 - Bug in :class:`DatetimeIndex` when passing an object-dtype ndarray of float objects and a ``tz`` incorrectly localizing the result (:issue:`55780`)
 - Bug in :func:`concat` raising ``AttributeError`` when concatenating all-NA DataFrame with :class:`DatetimeTZDtype` dtype DataFrame. (:issue:`52093`)
+- Bug in :func:`testing.assert_extension_array_equal` that could use the wrong unit when comparing resolutions (:issue:`55730`)
 - Bug in :func:`to_datetime` and :class:`DatetimeIndex` when passing a list of mixed-string-and-numeric types incorrectly raising (:issue:`55780`)
 - Bug in :meth:`DatetimeIndex.union` returning object dtype for tz-aware indexes with the same timezone but different units (:issue:`55238`)
 - Bug in :meth:`Index.is_monotonic_increasing` and :meth:`Index.is_monotonic_decreasing` always caching :meth:`Index.is_unique` as ``True`` when first value in index is ``NaT`` (:issue:`55755`)

environment.yml

@@ -16,7 +16,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-asyncio>=0.17.0
   - coverage
 
   # required dependencies

pandas/__init__.py

@@ -24,7 +24,7 @@
 try:
     # numpy compat
     from pandas.compat import (
-        is_numpy_dev as _is_numpy_dev,  # pyright: ignore[reportUnusedImport] # noqa: F401,E501
+        is_numpy_dev as _is_numpy_dev,  # pyright: ignore[reportUnusedImport] # noqa: F401
     )
 except ImportError as _err:  # pragma: no cover
     _module = _err.name

pandas/_libs/__init__.py

@@ -13,8 +13,8 @@
 # Below imports needs to happen first to ensure pandas top level
 # module gets monkeypatched with the pandas_datetime_CAPI
 # see pandas_datetime_exec in pd_datetime.c
-import pandas._libs.pandas_parser  # noqa: E501 # isort: skip # type: ignore[reportUnusedImport]
-import pandas._libs.pandas_datetime  # noqa: F401,E501 # isort: skip # type: ignore[reportUnusedImport]
+import pandas._libs.pandas_parser  # isort: skip # type: ignore[reportUnusedImport]
+import pandas._libs.pandas_datetime  # noqa: F401 # isort: skip # type: ignore[reportUnusedImport]
 from pandas._libs.interval import Interval
 from pandas._libs.tslibs import (
     NaT,

pandas/_libs/dtypes.pxd

@@ -34,3 +34,8 @@ ctypedef fused numeric_t:
 ctypedef fused numeric_object_t:
     numeric_t
     object
+
+ctypedef fused uint8_int64_object_t:
+    uint8_t
+    int64_t
+    object

pandas/_libs/include/pandas/portable.h

@@ -21,5 +21,5 @@ The full license is in the LICENSE file, distributed with this software.
 #define getdigit_ascii(c, default)                                             \
   (isdigit_ascii(c) ? ((int)((c) - '0')) : default)
 #define isspace_ascii(c) (((c) == ' ') || (((unsigned)(c) - '\t') < 5))
-#define toupper_ascii(c) ((((unsigned)(c) - 'a') < 26) ? ((c)&0x5f) : (c))
+#define toupper_ascii(c) ((((unsigned)(c) - 'a') < 26) ? ((c) & 0x5f) : (c))
 #define tolower_ascii(c) ((((unsigned)(c) - 'A') < 26) ? ((c) | 0x20) : (c))

pandas/_libs/include/pandas/vendored/ujson/lib/ultrajson.h

@@ -189,13 +189,13 @@ typedef struct __JSONObjectEncoder {
 
   /*
   Begin iteration of an iterable object (JS_ARRAY or JS_OBJECT)
-  Implementor should setup iteration state in ti->prv
+  Implementer should setup iteration state in ti->prv
   */
   JSPFN_ITERBEGIN iterBegin;
 
   /*
   Retrieve next object in an iteration. Should return 0 to indicate iteration
-  has reached end or 1 if there are more items. Implementor is responsible for
+  has reached end or 1 if there are more items. Implementer is responsible for
   keeping state of the iteration. Use ti->prv fields for this
   */
   JSPFN_ITERNEXT iterNext;

pandas/_libs/index.pyi

@@ -80,13 +80,6 @@ class BaseMultiIndexCodesEngine:
     ) -> None: ...
     def get_indexer(self, target: npt.NDArray[np.object_]) -> npt.NDArray[np.intp]: ...
     def _extract_level_codes(self, target: MultiIndex) -> np.ndarray: ...
-    def get_indexer_with_fill(
-        self,
-        target: np.ndarray,  # np.ndarray[object] of tuples
-        values: np.ndarray,  # np.ndarray[object] of tuples
-        method: str,
-        limit: int | None,
-    ) -> npt.NDArray[np.intp]: ...
 
 class ExtensionEngine:
     def __init__(self, values: ExtensionArray) -> None: ...