Merge branch 'main' into pandas-asan

Author: WillAyd

.github/workflows/codeql.yml

@@ -28,8 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: github/codeql-action/init@v2
+      - uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
-      - uses: github/codeql-action/autobuild@v2
-      - uses: github/codeql-action/analyze@v2
+      - uses: github/codeql-action/autobuild@v3
+      - uses: github/codeql-action/analyze@v3

.github/workflows/docbuild-and-upload.yml

@@ -85,7 +85,7 @@ jobs:
       run: mv doc/build/html web/build/docs
 
     - name: Save website as an artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: website
         path: web/build

.github/workflows/wheels.yml

@@ -62,7 +62,7 @@ jobs:
           python -m pip install build
           python -m build --sdist
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: sdist
           path: ./dist/*
@@ -115,7 +115,7 @@ jobs:
       # removes unnecessary files from the release
       - name: Download sdist (not macOS)
         #if: ${{ matrix.buildplat[1] != 'macosx_*' }}
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: sdist
           path: ./dist
@@ -189,7 +189,7 @@ jobs:
           docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
           docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl

.pre-commit-config.yaml

@@ -132,11 +132,11 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].318
+        - [email protected].339
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
-        entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json --level warning
+        entry: pyright -p pyright_reportGeneralTypeIssues.json --level warning
         language: node
         pass_filenames: false
         types: [python]

asv_bench/benchmarks/join_merge.py

@@ -275,18 +275,21 @@ def time_merge_dataframes_cross(self, sort):
 
 class MergeEA:
     params = [
-        "Int64",
-        "Int32",
-        "Int16",
-        "UInt64",
-        "UInt32",
-        "UInt16",
-        "Float64",
-        "Float32",
+        [
+            "Int64",
+            "Int32",
+            "Int16",
+            "UInt64",
+            "UInt32",
+            "UInt16",
+            "Float64",
+            "Float32",
+        ],
+        [True, False],
     ]
-    param_names = ["dtype"]
+    param_names = ["dtype", "monotonic"]
 
-    def setup(self, dtype):
+    def setup(self, dtype, monotonic):
         N = 10_000
         indices = np.arange(1, N)
         key = np.tile(indices[:8000], 10)
@@ -299,8 +302,11 @@ def setup(self, dtype):
                 "value2": np.random.randn(7999),
             }
         )
+        if monotonic:
+            self.left = self.left.sort_values("key")
+            self.right = self.right.sort_values("key")
 
-    def time_merge(self, dtype):
+    def time_merge(self, dtype, monotonic):
         merge(self.left, self.right)
 
 
@@ -330,10 +336,11 @@ class MergeDatetime:
             ("ns", "ms"),
         ],
         [None, "Europe/Brussels"],
+        [True, False],
     ]
-    param_names = ["units", "tz"]
+    param_names = ["units", "tz", "monotonic"]
 
-    def setup(self, units, tz):
+    def setup(self, units, tz, monotonic):
         unit_left, unit_right = units
         N = 10_000
         keys = Series(date_range("2012-01-01", freq="min", periods=N, tz=tz))
@@ -349,8 +356,11 @@ def setup(self, units, tz):
                 "value2": np.random.randn(8000),
             }
         )
+        if monotonic:
+            self.left = self.left.sort_values("key")
+            self.right = self.right.sort_values("key")
 
-    def time_merge(self, units, tz):
+    def time_merge(self, units, tz, monotonic):
         merge(self.left, self.right)
 
 

doc/source/conf.py

@@ -58,7 +58,6 @@
     "numpydoc",
     "sphinx_copybutton",
     "sphinx_design",
-    "sphinx_toggleprompt",
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
     "sphinx.ext.coverage",

doc/source/reference/groupby.rst

@@ -5,7 +5,7 @@
 =======
 GroupBy
 =======
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.groupby
 
 :class:`pandas.api.typing.DataFrameGroupBy` and :class:`pandas.api.typing.SeriesGroupBy`
 instances are returned by groupby calls :func:`pandas.DataFrame.groupby` and
@@ -40,7 +40,7 @@ Function application helper
 
    NamedAgg
 
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.groupby
 
 Function application
 --------------------

doc/source/reference/resampling.rst

@@ -5,7 +5,7 @@
 ==========
 Resampling
 ==========
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.resample
 
 :class:`pandas.api.typing.Resampler` instances are returned by
 resample calls: :func:`pandas.DataFrame.resample`, :func:`pandas.Series.resample`.

doc/source/reference/window.rst

@@ -17,7 +17,7 @@ calls: :func:`pandas.DataFrame.ewm` and :func:`pandas.Series.ewm`.
 
 Rolling window functions
 ------------------------
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.window.rolling
 
 .. autosummary::
    :toctree: api/
@@ -44,7 +44,7 @@ Rolling window functions
 
 Weighted window functions
 -------------------------
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.window.rolling
 
 .. autosummary::
    :toctree: api/
@@ -58,7 +58,7 @@ Weighted window functions
 
 Expanding window functions
 --------------------------
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.window.expanding
 
 .. autosummary::
    :toctree: api/
@@ -85,7 +85,7 @@ Expanding window functions
 
 Exponentially-weighted window functions
 ---------------------------------------
-.. currentmodule:: pandas.api.typing
+.. currentmodule:: pandas.core.window.ewm
 
 .. autosummary::
    :toctree: api/

doc/source/whatsnew/v0.21.0.rst

@@ -635,17 +635,22 @@ Previous behavior:
 
 New behavior:
 
-.. ipython:: python
+.. code-block:: ipython
 
-   pi = pd.period_range('2017-01', periods=12, freq='M')
+   In [1]: pi = pd.period_range('2017-01', periods=12, freq='M')
 
-   s = pd.Series(np.arange(12), index=pi)
+   In [2]: s = pd.Series(np.arange(12), index=pi)
 
-   resampled = s.resample('2Q').mean()
+   In [3]: resampled = s.resample('2Q').mean()
 
-   resampled
+   In [4]: resampled
+   Out[4]:
+   2017Q1    2.5
+   2017Q3    8.5
+   Freq: 2Q-DEC, dtype: float64
 
-   resampled.index
+   In [5]: resampled.index
+   Out[5]: PeriodIndex(['2017Q1', '2017Q3'], dtype='period[2Q-DEC]')
 
 Upsampling and calling ``.ohlc()`` previously returned a ``Series``, basically identical to calling ``.asfreq()``. OHLC upsampling now returns a DataFrame with columns ``open``, ``high``, ``low`` and ``close`` (:issue:`13083`). This is consistent with downsampling and ``DatetimeIndex`` behavior.
 

doc/source/whatsnew/v2.2.0.rst

@@ -229,6 +229,7 @@ Other enhancements
 - :func:`tseries.api.guess_datetime_format` is now part of the public API (:issue:`54727`)
 - :meth:`ExtensionArray._explode` interface method added to allow extension type implementations of the ``explode`` method (:issue:`54833`)
 - :meth:`ExtensionArray.duplicated` added to allow extension type implementations of the ``duplicated`` method (:issue:`55255`)
+- :meth:`Series.ffill`, :meth:`Series.bfill`, :meth:`DataFrame.ffill`, and :meth:`DataFrame.bfill` have gained the argument ``limit_area`` (:issue:`56492`)
 - Allow passing ``read_only``, ``data_only`` and ``keep_links`` arguments to openpyxl using ``engine_kwargs`` of :func:`read_excel` (:issue:`55027`)
 - DataFrame.apply now allows the usage of numba (via ``engine="numba"``) to JIT compile the passed function, allowing for potential speedups (:issue:`54666`)
 - Implement masked algorithms for :meth:`Series.value_counts` (:issue:`54984`)
@@ -345,6 +346,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | Package         | Minimum Version | Changed |
 +=================+=================+=========+
+| mypy (dev)      | 1.7.1           |    X    |
++-----------------+-----------------+---------+
 |                 |                 |    X    |
 +-----------------+-----------------+---------+
 
@@ -354,8 +357,8 @@ See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for mor
 
 Other API changes
 ^^^^^^^^^^^^^^^^^
+- The hash values of nullable extension dtypes changed to improve the performance of the hashing operation (:issue:`56507`)
 - ``check_exact`` now only takes effect for floating-point dtypes in :func:`testing.assert_frame_equal` and :func:`testing.assert_series_equal`. In particular, integer dtypes are always checked exactly (:issue:`55882`)
--
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_220.deprecations:
@@ -441,6 +444,7 @@ Other Deprecations
 - Deprecated :meth:`.DataFrameGroupBy.fillna` and :meth:`.SeriesGroupBy.fillna`; use :meth:`.DataFrameGroupBy.ffill`, :meth:`.DataFrameGroupBy.bfill` for forward and backward filling or :meth:`.DataFrame.fillna` to fill with a single value (or the Series equivalents) (:issue:`55718`)
 - Deprecated :meth:`Index.format`, use ``index.astype(str)`` or ``index.map(formatter)`` instead (:issue:`55413`)
 - Deprecated :meth:`Series.ravel`, the underlying array is already 1D, so ravel is not necessary (:issue:`52511`)
+- Deprecated :meth:`Series.resample` and :meth:`DataFrame.resample` with a :class:`PeriodIndex` (and the 'convention' keyword), convert to :class:`DatetimeIndex` (with ``.to_timestamp()``) before resampling instead (:issue:`53481`)
 - Deprecated :meth:`Series.view`, use :meth:`Series.astype` instead to change the dtype (:issue:`20251`)
 - Deprecated ``core.internals`` members ``Block``, ``ExtensionBlock``, and ``DatetimeTZBlock``, use public APIs instead (:issue:`55139`)
 - Deprecated ``year``, ``month``, ``quarter``, ``day``, ``hour``, ``minute``, and ``second`` keywords in the :class:`PeriodIndex` constructor, use :meth:`PeriodIndex.from_fields` instead (:issue:`55960`)
@@ -476,7 +480,7 @@ Other Deprecations
 - Deprecated strings ``H``, ``S``, ``U``, and ``N`` denoting units in :func:`to_timedelta` (:issue:`52536`)
 - Deprecated strings ``H``, ``T``, ``S``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`52536`)
 - Deprecated strings ``T``, ``S``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`52536`)
-- Deprecated the :class:`.BaseGrouper` attributes ``group_keys_seq`` and ``reconstructed_codes``; these will be removed in a future version of pandas (:issue:`56148`)
+- Deprecated the :attr:`.DataFrameGroupBy.grouper` and :attr:`SeriesGroupBy.grouper`; these attributes will be removed in a future version of pandas (:issue:`56521`)
 - Deprecated the :class:`.Grouping` attributes ``group_index``, ``result_index``, and ``group_arraylike``; these will be removed in a future version of pandas (:issue:`56148`)
 - Deprecated the ``errors="ignore"`` option in :func:`to_datetime`, :func:`to_timedelta`, and :func:`to_numeric`; explicitly catch exceptions instead (:issue:`54467`)
 - Deprecated the ``fastpath`` keyword in the :class:`Series` constructor (:issue:`20110`)
@@ -498,6 +502,7 @@ Performance improvements
 - Performance improvement in :func:`.testing.assert_frame_equal` and :func:`.testing.assert_series_equal` (:issue:`55949`, :issue:`55971`)
 - Performance improvement in :func:`concat` with ``axis=1`` and objects with unaligned indexes (:issue:`55084`)
 - Performance improvement in :func:`get_dummies` (:issue:`56089`)
+- Performance improvement in :func:`merge` and :func:`merge_ordered` when joining on sorted ascending keys (:issue:`56115`)
 - Performance improvement in :func:`merge_asof` when ``by`` is not ``None`` (:issue:`55580`, :issue:`55678`)
 - Performance improvement in :func:`read_stata` for files with many variables (:issue:`55515`)
 - Performance improvement in :meth:`DataFrame.groupby` when aggregating pyarrow timestamp and duration dtypes (:issue:`55031`)
@@ -514,6 +519,7 @@ Performance improvements
 - Performance improvement in :meth:`Series.value_counts` and :meth:`Series.mode` for masked dtypes (:issue:`54984`, :issue:`55340`)
 - Performance improvement in :meth:`.DataFrameGroupBy.nunique` and :meth:`.SeriesGroupBy.nunique` (:issue:`55972`)
 - Performance improvement in :meth:`.SeriesGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.DataFrameGroupBy.idxmin` (:issue:`54234`)
+- Performance improvement when hashing a nullable extension array (:issue:`56507`)
 - Performance improvement when indexing into a non-unique index (:issue:`55816`)
 - Performance improvement when indexing with more than 4 keys (:issue:`54550`)
 - Performance improvement when localizing time to UTC (:issue:`55241`)
@@ -658,11 +664,11 @@ Groupby/resample/rolling
 - Bug in :meth:`.DataFrameGroupBy.value_counts` and :meth:`.SeriesGroupBy.value_count` would sort by proportions rather than frequencies when ``sort=True`` and ``normalize=True`` (:issue:`55951`)
 - Bug in :meth:`DataFrame.asfreq` and :meth:`Series.asfreq` with a :class:`DatetimeIndex` with non-nanosecond resolution incorrectly converting to nanosecond resolution (:issue:`55958`)
 - Bug in :meth:`DataFrame.ewm` when passed ``times`` with non-nanosecond ``datetime64`` or :class:`DatetimeTZDtype` dtype (:issue:`56262`)
+- Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` where grouping by a combination of ``Decimal`` and NA values would fail when ``sort=True`` (:issue:`54847`)
 - Bug in :meth:`DataFrame.resample` not respecting ``closed`` and ``label`` arguments for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55282`)
 - Bug in :meth:`DataFrame.resample` when resampling on a :class:`ArrowDtype` of ``pyarrow.timestamp`` or ``pyarrow.duration`` type (:issue:`55989`)
 - Bug in :meth:`DataFrame.resample` where bin edges were not correct for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55281`)
 - Bug in :meth:`DataFrame.resample` where bin edges were not correct for :class:`~pandas.tseries.offsets.MonthBegin` (:issue:`55271`)
--
 
 Reshaping
 ^^^^^^^^^
@@ -671,10 +677,12 @@ Reshaping
 - Bug in :func:`merge_asof` raising ``TypeError`` when ``by`` dtype is not ``object``, ``int64``, or ``uint64`` (:issue:`22794`)
 - Bug in :func:`merge_asof` raising incorrect error for string dtype (:issue:`56444`)
 - Bug in :func:`merge_asof` when using a :class:`Timedelta` tolerance on a :class:`ArrowDtype` column (:issue:`56486`)
+- Bug in :func:`merge` not raising when merging string columns with numeric columns (:issue:`56441`)
 - Bug in :func:`merge` returning columns in incorrect order when left and/or right is empty (:issue:`51929`)
 - Bug in :meth:`DataFrame.melt` where an exception was raised if ``var_name`` was not a string (:issue:`55948`)
 - Bug in :meth:`DataFrame.melt` where it would not preserve the datetime (:issue:`55254`)
 - Bug in :meth:`DataFrame.pivot_table` where the row margin is incorrect when the columns have numeric names (:issue:`26568`)
+- Bug in :meth:`DataFrame.pivot` with numeric columns and extension dtype for data (:issue:`56528`)
 
 Sparse
 ^^^^^^
@@ -699,9 +707,10 @@ Other
 - Bug in :func:`infer_freq` and :meth:`DatetimeIndex.inferred_freq` with weekly frequencies and non-nanosecond resolutions (:issue:`55609`)
 - Bug in :meth:`DataFrame.apply` where passing ``raw=True`` ignored ``args`` passed to the applied function (:issue:`55009`)
 - Bug in :meth:`DataFrame.from_dict` which would always sort the rows of the created :class:`DataFrame`.  (:issue:`55683`)
+- Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` raising a ``ValueError`` (:issue:`56478`)
 - Bug in rendering ``inf`` values inside a a :class:`DataFrame` with the ``use_inf_as_na`` option enabled (:issue:`55483`)
 - Bug in rendering a :class:`Series` with a :class:`MultiIndex` when one of the index level's names is 0 not having that name displayed (:issue:`55415`)
-- Bug in the error message when assigning an empty dataframe to a column (:issue:`55956`)
+- Bug in the error message when assigning an empty :class:`DataFrame` to a column (:issue:`55956`)
 - Bug when time-like strings were being cast to :class:`ArrowDtype` with ``pyarrow.time64`` type (:issue:`56463`)
 
 .. ***DO NOT USE THIS SECTION***

environment.yml

@@ -68,17 +68,17 @@ dependencies:
   - flask
 
   # benchmarks
-  - asv>=0.5.1
+  - asv>=0.6.1
 
   ## The compiler packages are meta-packages and install the correct compiler (activation) packages on the respective platforms.
   - c-compiler
   - cxx-compiler
 
   # code checks
-  - flake8=6.0.0  # run in subprocess over docstring examples
-  - mypy=1.4.1  # pre-commit uses locally installed mypy
+  - flake8=6.1.0  # run in subprocess over docstring examples
+  - mypy=1.7.1  # pre-commit uses locally installed mypy
   - tokenize-rt  # scripts/check_for_inconsistent_pandas_namespace.py
-  - pre-commit>=2.15.0
+  - pre-commit>=3.6.0
 
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
@@ -98,12 +98,12 @@ dependencies:
   - types-setuptools
 
   # documentation (jupyter notebooks)
-  - nbconvert>=6.4.5
+  - nbconvert>=7.11.0
   - nbsphinx
   - pandoc
   - ipywidgets
   - nbformat
-  - notebook>=6.0.3
+  - notebook>=7.0.6
   - ipykernel
 
   # web
@@ -118,6 +118,5 @@ dependencies:
       - adbc-driver-postgresql>=0.8.0
       - adbc-driver-sqlite>=0.8.0
       - dataframe-api-compat>=0.1.7
-      - sphinx-toggleprompt  # conda-forge version has stricter pins on jinja2
       - typing_extensions; python_version<"3.11"
       - tzdata>=2022.7

pandas/_config/config.py

@@ -220,6 +220,8 @@ def get_default_val(pat: str):
 class DictWrapper:
     """provide attribute-style access to a nested dict"""
 
+    d: dict[str, Any]
+
     def __init__(self, d: dict[str, Any], prefix: str = "") -> None:
         object.__setattr__(self, "d", d)
         object.__setattr__(self, "prefix", prefix)
@@ -250,7 +252,7 @@ def __getattr__(self, key: str):
         else:
             return _get_option(prefix)
 
-    def __dir__(self) -> Iterable[str]:
+    def __dir__(self) -> list[str]:
         return list(self.d.keys())