Merge branch 'master' into faster_masked_transpose

Author: topper-123

.github/workflows/comment-commands.yml

@@ -15,21 +15,21 @@ jobs:
     concurrency:
       group: ${{ github.actor }}-issue-assign
     steps:
-      run: |
-        echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
-        curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+      - run: |
+          echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
   preview_docs:
     runs-on: ubuntu-22.04
     if: github.event.issue.pull_request && github.event.comment.body == '/preview'
     concurrency:
       group: ${{ github.actor }}-preview-docs
     steps:
-      run: |
-        if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
-          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
-        else
-          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
-        fi
+      - run: |
+          if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
+            curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
+          else
+            curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
+          fi
   asv_run:
     runs-on: ubuntu-22.04
     # TODO: Support more benchmarking options later, against different branches, against self, etc

ci/deps/actions-310.yaml

@@ -32,7 +32,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-311.yaml

@@ -32,7 +32,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   # - numba>=0.55.2 not compatible with 3.11
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-38-downstream_compat.yaml

@@ -33,7 +33,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-38.yaml

@@ -32,7 +32,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/actions-39.yaml

@@ -32,7 +32,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

ci/deps/circle-38-arm64.yaml

@@ -32,7 +32,7 @@ dependencies:
   - gcsfs>=2022.05.0
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   - numba>=0.55.2
   - numexpr>=2.8.0
   - odfpy>=1.4.1

doc/source/user_guide/io.rst

@@ -5239,6 +5239,7 @@ See the `Full Documentation <https://github.com/wesm/feather>`__.
 Write to a feather file.
 
 .. ipython:: python
+   :okwarning:
 
    df.to_feather("example.feather")
 
@@ -5382,6 +5383,7 @@ Serializing a ``DataFrame`` to parquet may include the implicit index as one or
 more columns in the output file. Thus, this code:
 
 .. ipython:: python
+   :okwarning:
 
     df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
     df.to_parquet("test.parquet", engine="pyarrow")
@@ -5398,6 +5400,7 @@ If you want to omit a dataframe's indexes when writing, pass ``index=False`` to
 :func:`~pandas.DataFrame.to_parquet`:
 
 .. ipython:: python
+   :okwarning:
 
     df.to_parquet("test.parquet", index=False)
 
@@ -5420,6 +5423,7 @@ Partitioning Parquet files
 Parquet supports partitioning of data based on the values of one or more columns.
 
 .. ipython:: python
+   :okwarning:
 
     df = pd.DataFrame({"a": [0, 0, 1, 1], "b": [0, 1, 0, 1]})
     df.to_parquet(path="test", engine="pyarrow", partition_cols=["a"], compression=None)

doc/source/user_guide/scale.rst

@@ -42,6 +42,7 @@ Suppose our raw dataset on disk has many columns::
 That can be generated by the following code snippet:
 
 .. ipython:: python
+   :okwarning:
 
    import pandas as pd
    import numpy as np
@@ -106,6 +107,7 @@ referred to as "low-cardinality" data). By using more efficient data types, you
 can store larger datasets in memory.
 
 .. ipython:: python
+   :okwarning:
 
    ts = make_timeseries(freq="30S", seed=0)
    ts.to_parquet("timeseries.parquet")
@@ -183,6 +185,7 @@ Suppose we have an even larger "logical dataset" on disk that's a directory of p
 files. Each file in the directory represents a different year of the entire dataset.
 
 .. ipython:: python
+   :okwarning:
 
    import pathlib
 

doc/source/whatsnew/v0.19.0.rst

@@ -905,6 +905,7 @@ As a consequence of this change, ``PeriodIndex`` no longer has an integer dtype:
 **New behavior**:
 
 .. ipython:: python
+   :okwarning:
 
    pi = pd.PeriodIndex(["2016-08-01"], freq="D")
    pi

doc/source/whatsnew/v2.0.1.rst

@@ -19,6 +19,7 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.sort_values` not resetting index when :class:`DataFrame` is already sorted and ``ignore_index=True`` (:issue:`52553`)
 - Fixed regression in :meth:`MultiIndex.isin` raising ``TypeError`` for ``Generator`` (:issue:`52568`)
 - Fixed regression in :meth:`Series.describe` showing ``RuntimeWarning`` for extension dtype :class:`Series` with one element (:issue:`52515`)
+- Fixed regression in :meth:`SeriesGroupBy.agg` failing when grouping with categorical data, multiple groupings, ``as_index=False``, and a list of aggregations (:issue:`52760`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_201.bug_fixes:
@@ -27,6 +28,8 @@ Bug fixes
 ~~~~~~~~~
 - Bug in :attr:`Series.dt.days` that would overflow ``int32`` number of days (:issue:`52391`)
 - Bug in :class:`arrays.DatetimeArray` constructor returning an incorrect unit when passed a non-nanosecond numpy datetime array (:issue:`52555`)
+- Bug in :class:`~arrays.ArrowExtensionArray` with duration dtype overflowing when constructed from data containing numpy ``NaT`` (:issue:`52843`)
+- Bug in :func:`Series.dt.round` when passing a ``freq`` of equal or higher resolution compared to the :class:`Series` would raise a ``ZeroDivisionError`` (:issue:`52761`)
 - Bug in :func:`Series.median` with :class:`ArrowDtype` returning an approximate median (:issue:`52679`)
 - Bug in :func:`api.interchange.from_dataframe` was unnecessarily raising on categorical dtypes (:issue:`49889`)
 - Bug in :func:`api.interchange.from_dataframe` was unnecessarily raising on large string dtypes (:issue:`52795`)
@@ -35,9 +38,11 @@ Bug fixes
 - Bug in :func:`to_datetime` and :func:`to_timedelta` when trying to convert numeric data with a :class:`ArrowDtype` (:issue:`52425`)
 - Bug in :func:`to_numeric` with ``errors='coerce'`` and ``dtype_backend='pyarrow'`` with :class:`ArrowDtype` data (:issue:`52588`)
 - Bug in :meth:`ArrowDtype.__from_arrow__` not respecting if dtype is explicitly given (:issue:`52533`)
+- Bug in :meth:`DataFrame.describe` not respecting ``ArrowDtype`` in ``include`` and ``exclude`` (:issue:`52570`)
 - Bug in :meth:`DataFrame.max` and related casting different :class:`Timestamp` resolutions always to nanoseconds (:issue:`52524`)
 - Bug in :meth:`Series.describe` not returning :class:`ArrowDtype` with ``pyarrow.float64`` type with numeric data (:issue:`52427`)
 - Bug in :meth:`Series.dt.tz_localize` incorrectly localizing timestamps with :class:`ArrowDtype` (:issue:`52677`)
+- Bug in arithmetic between ``np.datetime64`` and ``np.timedelta64`` ``NaT`` scalars with units always returning nanosecond resolution (:issue:`52295`)
 - Bug in logical and comparison operations between :class:`ArrowDtype` and numpy masked types (e.g. ``"boolean"``) (:issue:`52625`)
 - Fixed bug in :func:`merge` when merging with ``ArrowDtype`` one one and a NumPy dtype on the other side (:issue:`52406`)
 - Fixed segfault in :meth:`Series.to_numpy` with ``null[pyarrow]`` dtype (:issue:`52443`)
@@ -50,6 +55,7 @@ Other
 - :class:`DataFrame` created from empty dicts had :attr:`~DataFrame.columns`  of dtype ``object``. It is now a :class:`RangeIndex` (:issue:`52404`)
 - :class:`Series` created from empty dicts had :attr:`~Series.index`  of dtype ``object``. It is now a :class:`RangeIndex` (:issue:`52404`)
 - Implemented :meth:`Series.str.split` and :meth:`Series.str.rsplit` for :class:`ArrowDtype` with ``pyarrow.string`` (:issue:`52401`)
+- Implemented most ``str`` accessor methods for :class:`ArrowDtype` with ``pyarrow.string`` (:issue:`52401`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_201.contributors:

doc/source/whatsnew/v2.1.0.rst

@@ -87,6 +87,7 @@ Other enhancements
 - Added to the escape mode "latex-math" preserving without escaping all characters between "\(" and "\)" in formatter (:issue:`51903`)
 - Adding ``engine_kwargs`` parameter to :meth:`DataFrame.read_excel` (:issue:`52214`)
 - Classes that are useful for type-hinting have been added to the public API in the new submodule ``pandas.api.typing`` (:issue:`48577`)
+- Implemented :attr:`Series.dt.is_month_start`, :attr:`Series.dt.is_month_end`, :attr:`Series.dt.is_year_start`, :attr:`Series.dt.is_year_end`, :attr:`Series.dt.is_quarter_start`, :attr:`Series.dt.is_quarter_end`, :attr:`Series.dt.is_days_in_month`, :attr:`Series.dt.unit`, :meth:`Series.dt.is_normalize`, :meth:`Series.dt.day_name`, :meth:`Series.dt.month_name`, :meth:`Series.dt.tz_convert` for :class:`ArrowDtype` with ``pyarrow.timestamp`` (:issue:`52388`, :issue:`51718`)
 - Implemented ``__from_arrow__`` on :class:`DatetimeTZDtype`. (:issue:`52201`)
 - Implemented ``__pandas_priority__`` to allow custom types to take precedence over :class:`DataFrame`, :class:`Series`, :class:`Index`, or :class:`ExtensionArray` for arithmetic operations, :ref:`see the developer guide <extending.pandas_priority>` (:issue:`48347`)
 - Improve error message when having incompatible columns using :meth:`DataFrame.merge` (:issue:`51861`)
@@ -233,6 +234,8 @@ Deprecations
 - Deprecated :func:`is_datetime64tz_dtype`, check ``isinstance(dtype, pd.DatetimeTZDtype)`` instead (:issue:`52607`)
 - Deprecated :func:`is_int64_dtype`, check ``dtype == np.dtype(np.int64)`` instead (:issue:`52564`)
 - Deprecated :func:`is_interval_dtype`, check ``isinstance(dtype, pd.IntervalDtype)`` instead (:issue:`52607`)
+- Deprecated :func:`is_period_dtype`, check ``isinstance(dtype, pd.PeriodDtype)`` instead (:issue:`52642`)
+- Deprecated :func:`is_sparse`, check ``isinstance(dtype, pd.SparseDtype)`` instead (:issue:`52642`)
 - Deprecated :meth:`DataFrame.applymap`. Use the new :meth:`DataFrame.map` method instead (:issue:`52353`)
 - Deprecated :meth:`DataFrame.swapaxes` and :meth:`Series.swapaxes`, use :meth:`DataFrame.transpose` or :meth:`Series.transpose` instead (:issue:`51946`)
 - Deprecated ``freq`` parameter in :class:`PeriodArray` constructor, pass ``dtype`` instead (:issue:`52462`)

environment.yml

@@ -35,7 +35,7 @@ dependencies:
   - ipython
   - jinja2>=3.1.2
   - lxml>=4.8.0
-  - matplotlib>=3.6.1, <3.7.0
+  - matplotlib>=3.6.1
   - numba>=0.55.2
   - numexpr>=2.8.0  # pin for "Run checks on imported code" job
   - openpyxl<3.1.1, >=3.0.7

pandas/compat/numpy/__init__.py

@@ -8,6 +8,7 @@
 _nlv = Version(_np_version)
 np_version_under1p22 = _nlv < Version("1.22")
 np_version_gte1p24 = _nlv >= Version("1.24")
+np_version_gte1p24p3 = _nlv >= Version("1.24.3")
 is_numpy_dev = _nlv.dev is not None
 _min_numpy_ver = "1.21.6"
 

pandas/conftest.py

@@ -137,7 +137,10 @@ def pytest_collection_modifyitems(items, config) -> None:
     ignored_doctest_warnings = [
         ("is_int64_dtype", "is_int64_dtype is deprecated"),
         ("is_interval_dtype", "is_interval_dtype is deprecated"),
+        ("is_period_dtype", "is_period_dtype is deprecated"),
         ("is_datetime64tz_dtype", "is_datetime64tz_dtype is deprecated"),
+        ("is_categorical_dtype", "is_categorical_dtype is deprecated"),
+        ("is_sparse", "is_sparse is deprecated"),
         # Docstring divides by zero to show behavior difference
         ("missing.mask_zero_div_zero", "divide by zero encountered"),
         (
@@ -149,7 +152,6 @@ def pytest_collection_modifyitems(items, config) -> None:
             "(Series|DataFrame).bool is now deprecated and will be removed "
             "in future version of pandas",
         ),
-        ("is_categorical_dtype", "is_categorical_dtype is deprecated"),
     ]
 
     for item in items: