Merge remote-tracking branch 'upstream/main' into bodo_frame_apply

Author: datapythonista

.github/actions/run-tests/action.yml

@@ -14,7 +14,7 @@ runs:
       if: failure()
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         flags: unittests
         name: codecov-pandas

.github/actions/setup-conda/action.yml

@@ -7,7 +7,7 @@ runs:
   using: composite
   steps:
     - name: Install ${{ inputs.environment-file }}
-      uses: mamba-org/setup-micromamba@v1
+      uses: mamba-org/setup-micromamba@v2
       with:
         environment-file: ${{ inputs.environment-file }}
         environment-name: test

.github/workflows/docbuild-and-upload.yml

@@ -59,6 +59,10 @@ jobs:
     - name: Build documentation
       run: doc/make.py --warnings-are-errors
 
+    - name: Build the interactive terminal
+      working-directory: web/interactive_terminal
+      run: jupyter lite build
+
     - name: Build documentation zip
       run: doc/make.py zip_html
 

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.4
+    rev: v0.9.9
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -70,7 +70,7 @@ repos:
     -   id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
 -   repo: https://github.com/PyCQA/isort
-    rev: 6.0.0
+    rev: 6.0.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade

Dockerfile

@@ -13,5 +13,5 @@ COPY requirements-dev.txt /tmp
 RUN python -m pip install -r /tmp/requirements-dev.txt
 RUN git config --global --add safe.directory /home/pandas
 
-ENV SHELL "/bin/bash"
+ENV SHELL="/bin/bash"
 CMD ["/bin/bash"]

asv_bench/benchmarks/rolling.py

@@ -10,7 +10,19 @@ class Methods:
         ["DataFrame", "Series"],
         [("rolling", {"window": 10}), ("rolling", {"window": 1000}), ("expanding", {})],
         ["int", "float"],
-        ["median", "mean", "max", "min", "std", "count", "skew", "kurt", "sum", "sem"],
+        [
+            "median",
+            "mean",
+            "max",
+            "min",
+            "std",
+            "count",
+            "skew",
+            "kurt",
+            "sum",
+            "sem",
+            "nunique",
+        ],
     )
     param_names = ["constructor", "window_kwargs", "dtype", "method"]
 

ci/code_checks.sh

@@ -83,6 +83,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.n GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.normalize GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.rule_code GL08" \
+        -i "pandas.tseries.offsets.BHalfYearBegin.startingMonth GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.n GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.normalize GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.rule_code GL08" \
+        -i "pandas.tseries.offsets.BHalfYearEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.n GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.normalize GL08" \
@@ -185,6 +195,16 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.FY5253Quarter.variation GL08" \
         -i "pandas.tseries.offsets.FY5253Quarter.weekday GL08" \
         -i "pandas.tseries.offsets.FY5253Quarter.year_has_extra_week GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.n GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.normalize GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.rule_code GL08" \
+        -i "pandas.tseries.offsets.HalfYearBegin.startingMonth GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.is_on_offset GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.n GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.normalize GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.rule_code GL08" \
+        -i "pandas.tseries.offsets.HalfYearEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.Hour.is_on_offset GL08" \
         -i "pandas.tseries.offsets.Hour.n GL08" \
         -i "pandas.tseries.offsets.Hour.normalize GL08" \

doc/source/getting_started/overview.rst

@@ -174,3 +174,4 @@ License
 -------
 
 .. literalinclude:: ../../../LICENSE
+   :language: none

doc/source/reference/offset_frequency.rst

@@ -776,6 +776,146 @@ Methods
     QuarterBegin.is_year_start
     QuarterBegin.is_year_end
 
+BHalfYearEnd
+------------
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd.freqstr
+    BHalfYearEnd.kwds
+    BHalfYearEnd.name
+    BHalfYearEnd.nanos
+    BHalfYearEnd.normalize
+    BHalfYearEnd.rule_code
+    BHalfYearEnd.n
+    BHalfYearEnd.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearEnd.copy
+    BHalfYearEnd.is_on_offset
+    BHalfYearEnd.is_month_start
+    BHalfYearEnd.is_month_end
+    BHalfYearEnd.is_quarter_start
+    BHalfYearEnd.is_quarter_end
+    BHalfYearEnd.is_year_start
+    BHalfYearEnd.is_year_end
+
+BHalfYearBegin
+--------------
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin.freqstr
+    BHalfYearBegin.kwds
+    BHalfYearBegin.name
+    BHalfYearBegin.nanos
+    BHalfYearBegin.normalize
+    BHalfYearBegin.rule_code
+    BHalfYearBegin.n
+    BHalfYearBegin.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    BHalfYearBegin.copy
+    BHalfYearBegin.is_on_offset
+    BHalfYearBegin.is_month_start
+    BHalfYearBegin.is_month_end
+    BHalfYearBegin.is_quarter_start
+    BHalfYearBegin.is_quarter_end
+    BHalfYearBegin.is_year_start
+    BHalfYearBegin.is_year_end
+
+HalfYearEnd
+-----------
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd.freqstr
+    HalfYearEnd.kwds
+    HalfYearEnd.name
+    HalfYearEnd.nanos
+    HalfYearEnd.normalize
+    HalfYearEnd.rule_code
+    HalfYearEnd.n
+    HalfYearEnd.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearEnd.copy
+    HalfYearEnd.is_on_offset
+    HalfYearEnd.is_month_start
+    HalfYearEnd.is_month_end
+    HalfYearEnd.is_quarter_start
+    HalfYearEnd.is_quarter_end
+    HalfYearEnd.is_year_start
+    HalfYearEnd.is_year_end
+
+HalfYearBegin
+-------------
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin
+
+Properties
+~~~~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin.freqstr
+    HalfYearBegin.kwds
+    HalfYearBegin.name
+    HalfYearBegin.nanos
+    HalfYearBegin.normalize
+    HalfYearBegin.rule_code
+    HalfYearBegin.n
+    HalfYearBegin.startingMonth
+
+Methods
+~~~~~~~
+.. autosummary::
+   :toctree: api/
+
+    HalfYearBegin.copy
+    HalfYearBegin.is_on_offset
+    HalfYearBegin.is_month_start
+    HalfYearBegin.is_month_end
+    HalfYearBegin.is_quarter_start
+    HalfYearBegin.is_quarter_end
+    HalfYearBegin.is_year_start
+    HalfYearBegin.is_year_end
+
 BYearEnd
 --------
 .. autosummary::

doc/source/reference/window.rst

@@ -42,6 +42,7 @@ Rolling window functions
    Rolling.quantile
    Rolling.sem
    Rolling.rank
+   Rolling.nunique
 
 .. _api.functions_window:
 
@@ -86,6 +87,7 @@ Expanding window functions
    Expanding.quantile
    Expanding.sem
    Expanding.rank
+   Expanding.nunique
 
 .. _api.functions_ewm:
 

doc/source/user_guide/io.rst

@@ -18,10 +18,10 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     :widths: 30, 100, 60, 60
 
     text,`CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`__, :ref:`read_csv<io.read_csv_table>`, :ref:`to_csv<io.store_in_csv>`
-    text,Fixed-Width Text File, :ref:`read_fwf<io.fwf_reader>` , NA
+    text,Fixed-Width Text File, :ref:`read_fwf<io.fwf_reader>`, NA
     text,`JSON <https://www.json.org/>`__, :ref:`read_json<io.json_reader>`, :ref:`to_json<io.json_writer>`
     text,`HTML <https://en.wikipedia.org/wiki/HTML>`__, :ref:`read_html<io.read_html>`, :ref:`to_html<io.html>`
-    text,`LaTeX <https://en.wikipedia.org/wiki/LaTeX>`__, :ref:`Styler.to_latex<io.latex>` , NA
+    text,`LaTeX <https://en.wikipedia.org/wiki/LaTeX>`__, NA, :ref:`Styler.to_latex<io.latex>`
     text,`XML <https://www.w3.org/standards/xml/core>`__, :ref:`read_xml<io.read_xml>`, :ref:`to_xml<io.xml>`
     text, Local clipboard, :ref:`read_clipboard<io.clipboard>`, :ref:`to_clipboard<io.clipboard>`
     binary,`MS Excel <https://en.wikipedia.org/wiki/Microsoft_Excel>`__ , :ref:`read_excel<io.excel_reader>`, :ref:`to_excel<io.excel_writer>`

doc/source/user_guide/text.rst

@@ -90,7 +90,7 @@ Behavior differences
 These are places where the behavior of ``StringDtype`` objects differ from
 ``object`` dtype:
 
-l. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
+1. For ``StringDtype``, :ref:`string accessor methods<api.series.str>`
    that return **numeric** output will always return a nullable integer dtype,
    rather than either int or float dtype, depending on the presence of NA values.
    Methods returning **boolean** output will return a nullable boolean dtype.
@@ -332,8 +332,8 @@ regular expression object will raise a ``ValueError``.
     ---------------------------------------------------------------------------
     ValueError: case and flags cannot be set when pat is a compiled regex
 
-``removeprefix`` and ``removesuffix`` have the same effect as ``str.removeprefix`` and ``str.removesuffix`` added in Python 3.9
-<https://docs.python.org/3/library/stdtypes.html#str.removeprefix>`__:
+``removeprefix`` and ``removesuffix`` have the same effect as ``str.removeprefix`` and ``str.removesuffix`` added in
+`Python 3.9 <https://docs.python.org/3/library/stdtypes.html#str.removeprefix>`__:
 
 .. versionadded:: 1.4.0
 

doc/source/user_guide/timeseries.rst

@@ -891,6 +891,10 @@ into ``freq`` keyword arguments. The available date offsets and associated frequ
     :class:`~pandas.tseries.offsets.BQuarterEnd`, ``'BQE``, "business quarter end"
     :class:`~pandas.tseries.offsets.BQuarterBegin`, ``'BQS'``, "business quarter begin"
     :class:`~pandas.tseries.offsets.FY5253Quarter`, ``'REQ'``, "retail (aka 52-53 week) quarter"
+    :class:`~pandas.tseries.offsets.HalfYearEnd`, ``'HYE'``, "calendar half year end"
+    :class:`~pandas.tseries.offsets.HalfYearBegin`, ``'HYS'``, "calendar half year begin"
+    :class:`~pandas.tseries.offsets.BHalfYearEnd`, ``'BHYE``, "business half year end"
+    :class:`~pandas.tseries.offsets.BHalfYearBegin`, ``'BHYS'``, "business half year begin"
     :class:`~pandas.tseries.offsets.YearEnd`, ``'YE'``, "calendar year end"
     :class:`~pandas.tseries.offsets.YearBegin`, ``'YS'`` or ``'BYS'``,"calendar year begin"
     :class:`~pandas.tseries.offsets.BYearEnd`, ``'BYE'``, "business year end"

doc/source/whatsnew/v2.3.0.rst

@@ -118,6 +118,7 @@ Conversion
 
 Strings
 ^^^^^^^
+- Bug in :meth:`DataFrame.sum` with ``axis=1``, :meth:`.DataFrameGroupBy.sum` or :meth:`.SeriesGroupBy.sum` with ``skipna=True``, and :meth:`.Resampler.sum` on :class:`StringDtype` with all NA values resulted in ``0`` and is now the empty string ``""`` (:issue:`60229`)
 - Bug in :meth:`Series.__pos__` and :meth:`DataFrame.__pos__` did not raise for :class:`StringDtype` with ``storage="pyarrow"`` (:issue:`60710`)
 - Bug in :meth:`Series.rank` for :class:`StringDtype` with ``storage="pyarrow"`` incorrectly returning integer results in case of ``method="average"`` and raising an error if it would truncate results (:issue:`59768`)
 - Bug in :meth:`Series.replace` with :class:`StringDtype` when replacing with a non-string value was not upcasting to ``object`` dtype (:issue:`60282`)

doc/source/whatsnew/v3.0.0.rst

@@ -62,6 +62,7 @@ Other enhancements
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
 - :meth:`DataFrame.plot.scatter` argument ``c`` now accepts a column of strings, where rows with the same string are colored identically (:issue:`16827` and :issue:`16485`)
 - :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` methods ``sum``, ``mean``, ``median``, ``prod``, ``min``, ``max``, ``std``, ``var`` and ``sem`` now accept ``skipna`` parameter (:issue:`15675`)
+- :class:`Rolling` and :class:`Expanding` now support ``nunique`` (:issue:`26958`)
 - :class:`Rolling` and :class:`Expanding` now support aggregations ``first`` and ``last`` (:issue:`33155`)
 - :func:`read_parquet` accepts ``to_pandas_kwargs`` which are forwarded to :meth:`pyarrow.Table.to_pandas` which enables passing additional keywords to customize the conversion to pandas, such as ``maps_as_pydicts`` to read the Parquet map data type as python dictionaries (:issue:`56842`)
 - :meth:`.DataFrameGroupBy.transform`, :meth:`.SeriesGroupBy.transform`, :meth:`.DataFrameGroupBy.agg`, :meth:`.SeriesGroupBy.agg`, :meth:`.SeriesGroupBy.apply`, :meth:`.DataFrameGroupBy.apply` now support ``kurt`` (:issue:`40139`)
@@ -73,8 +74,10 @@ Other enhancements
 - :meth:`pandas.concat` will raise a ``ValueError`` when ``ignore_index=True`` and ``keys`` is not ``None`` (:issue:`59274`)
 - :py:class:`frozenset` elements in pandas objects are now natively printed (:issue:`60690`)
 - Add ``"delete_rows"`` option to ``if_exists`` argument in :meth:`DataFrame.to_sql` deleting all records of the table before inserting data (:issue:`37210`).
+- Added half-year offset classes :class:`HalfYearBegin`, :class:`HalfYearEnd`, :class:`BHalfYearBegin` and :class:`BHalfYearEnd` (:issue:`60928`)
 - Errors occurring during SQL I/O will now throw a generic :class:`.DatabaseError` instead of the raw Exception type from the underlying driver manager library (:issue:`60748`)
 - Implemented :meth:`Series.str.isascii` and :meth:`Series.str.isascii` (:issue:`59091`)
+- Improved deprecation message for offset aliases (:issue:`60820`)
 - Multiplying two :class:`DateOffset` objects will now raise a ``TypeError`` instead of a ``RecursionError`` (:issue:`59442`)
 - Restore support for reading Stata 104-format and enable reading 103-format dta files (:issue:`58554`)
 - Support passing a :class:`Iterable[Hashable]` input to :meth:`DataFrame.drop_duplicates` (:issue:`59237`)
@@ -693,8 +696,10 @@ Interval
 Indexing
 ^^^^^^^^
 - Bug in :meth:`DataFrame.__getitem__` returning modified columns when called with ``slice`` in Python 3.12 (:issue:`57500`)
+- Bug in :meth:`DataFrame.__getitem__` when slicing a :class:`DataFrame` with many rows raised an ``OverflowError`` (:issue:`59531`)
 - Bug in :meth:`DataFrame.from_records` throwing a ``ValueError`` when passed an empty list in ``index`` (:issue:`58594`)
 - Bug in :meth:`DataFrame.loc` with inconsistent behavior of loc-set with 2 given indexes to Series (:issue:`59933`)
+- Bug in :meth:`Index.get_indexer` and similar methods when ``NaN`` is located at or after position 128 (:issue:`58924`)
 - Bug in :meth:`MultiIndex.insert` when a new value inserted to a datetime-like level gets cast to ``NaT`` and fails indexing (:issue:`60388`)
 - Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)
 
@@ -711,12 +716,13 @@ MultiIndex
 - :func:`MultiIndex.get_level_values` accessing a :class:`DatetimeIndex` does not carry the frequency attribute along (:issue:`58327`, :issue:`57949`)
 - Bug in :class:`DataFrame` arithmetic operations in case of unaligned MultiIndex columns (:issue:`60498`)
 - Bug in :class:`DataFrame` arithmetic operations with :class:`Series` in case of unaligned MultiIndex (:issue:`61009`)
--
+- Bug in :meth:`MultiIndex.from_tuples` causing wrong output with input of type tuples having NaN values (:issue:`60695`, :issue:`60988`)
 
 I/O
 ^^^
 - Bug in :class:`DataFrame` and :class:`Series` ``repr`` of :py:class:`collections.abc.Mapping`` elements. (:issue:`57915`)
 - Bug in :meth:`.DataFrame.to_json` when ``"index"`` was a value in the :attr:`DataFrame.column` and :attr:`Index.name` was ``None``. Now, this will fail with a ``ValueError`` (:issue:`58925`)
+- Bug in :meth:`.io.common.is_fsspec_url` not recognizing chained fsspec URLs (:issue:`48978`)
 - Bug in :meth:`DataFrame._repr_html_` which ignored the ``"display.float_format"`` option (:issue:`59876`)
 - Bug in :meth:`DataFrame.from_records` where ``columns`` parameter with numpy structured array was not reordering and filtering out the columns (:issue:`59717`)
 - Bug in :meth:`DataFrame.to_dict` raises unnecessary ``UserWarning`` when columns are not unique and ``orient='tight'``. (:issue:`58281`)
@@ -785,6 +791,7 @@ Reshaping
 - Bug in :meth:`DataFrame.pivot_table` incorrectly subaggregating results when called without an ``index`` argument (:issue:`58722`)
 - Bug in :meth:`DataFrame.stack` with the new implementation where ``ValueError`` is raised when ``level=[]`` (:issue:`60740`)
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when manipulating empty :class:`DataFrame` with an :class:`ExtentionDtype` (:issue:`59123`)
+- Bug in :meth:`concat` where concatenating DataFrame and Series with ``ignore_index = True`` drops the series name (:issue:`60723`, :issue:`56257`)
 
 Sparse
 ^^^^^^

environment.yml

@@ -116,6 +116,13 @@ dependencies:
   - requests
   - pygments # Code highlighting
 
+  # web interactive REPL
+  # see the following links for more context:
+  # 1. https://jupyterlite-pyodide-kernel.readthedocs.io/en/stable/#compatibility
+  # 2. https://pyodide.org/en/stable/usage/packages-in-pyodide.html
+  - jupyterlite-core
+  - jupyterlite-pyodide-kernel
+
   - pip:
       - adbc-driver-postgresql>=0.10.0
       - adbc-driver-sqlite>=0.8.0