Merge branch 'main' into issue#26195

Author: kvn4

.circleci/config.yml

@@ -47,8 +47,8 @@ jobs:
       - run:
           name: Build aarch64 wheels
           command: |
-            pip3 install cibuildwheel==2.12.1
-            cibuildwheel --output-dir wheelhouse
+            pip3 install cibuildwheel==2.14.1
+            cibuildwheel --prerelease-pythons --output-dir wheelhouse
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
 
@@ -91,4 +91,5 @@ workflows:
               only: /^v.*/
           matrix:
             parameters:
-              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"]
+              # TODO: Enable Python 3.12 wheels when numpy releases a version that supports Python 3.12
+              cibw-build: ["cp39-manylinux_aarch64", "cp310-manylinux_aarch64", "cp311-manylinux_aarch64"]#, "cp312-manylinux_aarch64"]

.github/CODEOWNERS

@@ -9,6 +9,7 @@ web/                              @datapythonista
 
 # docs
 doc/cheatsheet                    @Dr-Irv
+doc/source/development            @noatamir
 
 # pandas
 pandas/_libs/                     @WillAyd

.github/workflows/unit-tests.yml

@@ -103,7 +103,7 @@ jobs:
 
     services:
       mysql:
-        image: mysql
+        image: mysql:8.0.33
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: yes
           MYSQL_DATABASE: pandas
@@ -116,8 +116,9 @@ jobs:
           - 3306:3306
 
       postgres:
-        image: postgres
+        image: postgres:13
         env:
+          PGUSER: postgres
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: pandas
@@ -130,7 +131,7 @@ jobs:
           - 5432:5432
 
       moto:
-        image: motoserver/moto:4.1.12
+        image: motoserver/moto:4.1.13
         env:
           AWS_ACCESS_KEY_ID: foobar_key
           AWS_SECRET_ACCESS_KEY: foobar_secret
@@ -237,7 +238,7 @@ jobs:
         run: |
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
@@ -275,7 +276,7 @@ jobs:
         run: |
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
@@ -310,12 +311,16 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    #if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-latest, windows-latest]
+        # TODO: Disable macOS for now, Github Actions bug where python is not
+        # symlinked correctly to 3.12
+        # xref https://github.com/actions/setup-python/issues/701
+        #os: [ubuntu-22.04, macOS-latest, windows-latest]
+        os: [ubuntu-22.04, windows-latest]
 
     timeout-minutes: 180
 
@@ -339,21 +344,21 @@ jobs:
       - name: Set up Python Dev Version
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11-dev'
+          python-version: '3.12-dev'
 
       - name: Install dependencies
         run: |
           python --version
-          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.0.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install git+https://github.com/nedbat/coveragepy.git
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
+          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip list
 
       - name: Build Pandas
         run: |
-          python -m pip install -e . --no-build-isolation --no-index
+          python -m pip install -ve . --no-build-isolation --no-index
 
       - name: Build Version
         run: |

.github/workflows/wheels.yml

@@ -93,7 +93,8 @@ jobs:
         - [macos-12, macosx_*]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
-        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]
+        # TODO: Enable Python 3.12 wheels when numpy releases a version that supports Python 3.12
+        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]#, ["cp312", "3.12"]]
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
       IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
@@ -117,6 +118,7 @@ jobs:
         #with:
         #  package-dir: ./dist/${{ needs.build_sdist.outputs.sdist_file }}
         env:
+          CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       - name: Set up Python

CITATION.cff

@@ -3,8 +3,12 @@ title: 'pandas-dev/pandas: Pandas'
 message: 'If you use this software, please cite it as below.'
 authors:
   - name: "The pandas development team"
+abstract: "Pandas is a powerful data structures for data analysis, time series, and statistics."
 license: BSD-3-Clause
 license-url: "https://github.com/pandas-dev/pandas/blob/main/LICENSE"
 repository-code: "https://github.com/pandas-dev/pandas"
+keywords:
+  - python
+  - data science
 type: software
 url: "https://github.com/pandas-dev/pandas"

asv_bench/benchmarks/frame_methods.py

@@ -565,10 +565,7 @@ def time_frame_object_unequal(self):
 
 
 class Interpolate:
-    params = [None, "infer"]
-    param_names = ["downcast"]
-
-    def setup(self, downcast):
+    def setup(self):
         N = 10000
         # this is the worst case, where every column has NaNs.
         arr = np.random.randn(N, 100)
@@ -589,11 +586,11 @@ def setup(self, downcast):
         self.df2.loc[1::5, "A"] = np.nan
         self.df2.loc[1::5, "C"] = np.nan
 
-    def time_interpolate(self, downcast):
-        self.df.interpolate(downcast=downcast)
+    def time_interpolate(self):
+        self.df.interpolate()
 
-    def time_interpolate_some_good(self, downcast):
-        self.df2.interpolate(downcast=downcast)
+    def time_interpolate_some_good(self):
+        self.df2.interpolate()
 
 
 class Shift:

asv_bench/benchmarks/io/csv.py

@@ -341,7 +341,7 @@ def setup(self, sep, thousands, engine):
         if thousands is not None:
             fmt = f":{thousands}"
             fmt = "{" + fmt + "}"
-            df = df.applymap(lambda x: fmt.format(x))
+            df = df.map(lambda x: fmt.format(x))
         df.to_csv(self.fname, sep=sep)
 
     def time_thousands(self, sep, thousands, engine):

asv_bench/benchmarks/io/excel.py

@@ -57,9 +57,9 @@ def time_write_excel_style(self, engine):
         bio.seek(0)
         with ExcelWriter(bio, engine=engine) as writer:
             df_style = self.df.style
-            df_style.applymap(lambda x: "border: red 1px solid;")
-            df_style.applymap(lambda x: "color: blue")
-            df_style.applymap(lambda x: "border-color: green black", subset=["float1"])
+            df_style.map(lambda x: "border: red 1px solid;")
+            df_style.map(lambda x: "color: blue")
+            df_style.map(lambda x: "border-color: green black", subset=["float1"])
             df_style.to_excel(writer, sheet_name="Sheet1")
 
 

asv_bench/benchmarks/io/style.py

@@ -66,7 +66,7 @@ def _apply_func(s):
         self.st = self.df.style.apply(_apply_func, axis=1)
 
     def _style_classes(self):
-        classes = self.df.applymap(lambda v: ("cls-1" if v > 0 else ""))
+        classes = self.df.map(lambda v: ("cls-1" if v > 0 else ""))
         classes.index, classes.columns = self.df.index, self.df.columns
         self.st = self.df.style.set_td_classes(classes)
 
@@ -80,7 +80,7 @@ def _style_format(self):
         )
 
     def _style_apply_format_hide(self):
-        self.st = self.df.style.applymap(lambda v: "color: red;")
+        self.st = self.df.style.map(lambda v: "color: red;")
         self.st.format("{:.3f}")
         self.st.hide(self.st.index[1:], axis=0)
         self.st.hide(self.st.columns[1:], axis=1)

ci/code_checks.sh

@@ -63,74 +63,26 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
     MSG='Partially validate docstrings (EX01)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX01 --ignore_functions \
-        pandas.errors.IncompatibilityWarning \
-        pandas.errors.InvalidComparison \
-        pandas.errors.IntCastingNaNError \
-        pandas.errors.LossySetitemError \
-        pandas.errors.MergeError \
-        pandas.errors.NoBufferPresent \
-        pandas.errors.NullFrequencyError \
-        pandas.errors.NumbaUtilError \
-        pandas.errors.OptionError \
-        pandas.errors.OutOfBoundsDatetime \
-        pandas.errors.OutOfBoundsTimedelta \
-        pandas.errors.ParserError \
         pandas.errors.PerformanceWarning \
         pandas.errors.PyperclipException \
         pandas.errors.PyperclipWindowsException \
         pandas.errors.UnsortedIndexError \
         pandas.errors.UnsupportedFunctionCall \
-        pandas.test \
         pandas.NaT \
-        pandas.read_feather \
-        pandas.DataFrame.to_feather \
-        pandas.read_orc \
-        pandas.read_sas \
-        pandas.read_spss \
-        pandas.read_sql_query \
         pandas.io.stata.StataReader.data_label \
         pandas.io.stata.StataReader.value_labels \
         pandas.io.stata.StataReader.variable_labels \
         pandas.io.stata.StataWriter.write_file \
         pandas.plotting.deregister_matplotlib_converters \
-        pandas.plotting.plot_params \
         pandas.plotting.register_matplotlib_converters \
-        pandas.plotting.table \
-        pandas.util.hash_array \
-        pandas.util.hash_pandas_object \
-        pandas_object \
-        pandas.api.interchange.from_dataframe \
-        pandas.DatetimeIndex.snap \
-        pandas.api.indexers.BaseIndexer \
-        pandas.api.indexers.VariableOffsetWindowIndexer \
         pandas.api.extensions.ExtensionDtype \
         pandas.api.extensions.ExtensionArray \
         pandas.arrays.NumpyExtensionArray \
-        pandas.api.extensions.ExtensionArray._accumulate \
-        pandas.api.extensions.ExtensionArray._concat_same_type \
-        pandas.api.extensions.ExtensionArray._formatter \
-        pandas.api.extensions.ExtensionArray._from_factorized \
-        pandas.api.extensions.ExtensionArray._from_sequence \
         pandas.api.extensions.ExtensionArray._from_sequence_of_strings \
         pandas.api.extensions.ExtensionArray._hash_pandas_object \
         pandas.api.extensions.ExtensionArray._reduce \
         pandas.api.extensions.ExtensionArray._values_for_factorize \
-        pandas.api.extensions.ExtensionArray.dropna \
-        pandas.api.extensions.ExtensionArray.equals \
-        pandas.api.extensions.ExtensionArray.factorize \
-        pandas.api.extensions.ExtensionArray.fillna \
-        pandas.api.extensions.ExtensionArray.insert \
         pandas.api.extensions.ExtensionArray.interpolate \
-        pandas.api.extensions.ExtensionArray.isin \
-        pandas.api.extensions.ExtensionArray.isna \
-        pandas.api.extensions.ExtensionArray.ravel \
-        pandas.api.extensions.ExtensionArray.searchsorted \
-        pandas.api.extensions.ExtensionArray.shift \
-        pandas.api.extensions.ExtensionArray.unique \
-        pandas.api.extensions.ExtensionArray.ndim \
-        pandas.api.extensions.ExtensionArray.shape \
-        pandas.api.extensions.ExtensionArray.tolist \
-        pandas.DataFrame.__dataframe__
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/meta.yaml

@@ -34,7 +34,8 @@ requirements:
     - tomli  # [py<311]
   run:
     - python
-    - {{ pin_compatible('numpy') }}
+    - numpy >=1.21.6  # [py<311]
+    - numpy >=1.23.2  # [py>=311]
     - python-dateutil >=2.8.2
     - pytz >=2020.1
     - python-tzdata >=2022.1

doc/source/conf.py

@@ -348,10 +348,8 @@
     methods = [
         x for x in dir(klass) if not x.startswith("_") or x in ("__iter__", "__array__")
     ]
-
-    for method in methods:
-        # ... and each of its public methods
-        moved_api_pages.append((f"{old}.{method}", f"{new}.{method}"))
+    # ... and each of its public methods
+    moved_api_pages.extend((f"{old}.{method}", f"{new}.{method}") for method in methods)
 
 if include_api:
     html_additional_pages = {

doc/source/user_guide/categorical.rst

@@ -779,6 +779,7 @@ Setting values by assigning categorical data will also check that the ``categori
 Assigning a ``Categorical`` to parts of a column of other types will use the values:
 
 .. ipython:: python
+    :okwarning:
 
     df = pd.DataFrame({"a": [1, 1, 1, 1, 1], "b": ["a", "a", "a", "a", "a"]})
     df.loc[1:2, "a"] = pd.Categorical(["b", "b"], categories=["a", "b"])

doc/source/user_guide/indexing.rst

@@ -155,7 +155,7 @@ columns.
 
 .. warning::
 
-   pandas aligns all AXES when setting ``Series`` and ``DataFrame`` from ``.loc``, and ``.iloc``.
+   pandas aligns all AXES when setting ``Series`` and ``DataFrame`` from ``.loc``.
 
    This will **not** modify ``df`` because the column alignment is before value assignment.
 
@@ -172,6 +172,17 @@ columns.
       df.loc[:, ['B', 'A']] = df[['A', 'B']].to_numpy()
       df[['A', 'B']]
 
+   However, pandas does not align AXES when setting ``Series`` and ``DataFrame`` from ``.iloc``
+   because ``.iloc`` operates by position.
+
+   This will modify ``df`` because the column alignment is not done before value assignment.
+
+   .. ipython:: python
+
+      df[['A', 'B']]
+      df.iloc[:, [1, 0]] = df[['A', 'B']]
+      df[['A','B']]
+
 
 Attribute access
 ----------------

doc/source/user_guide/io.rst

@@ -931,6 +931,8 @@ Parsing a CSV with mixed timezones
 pandas cannot natively represent a column or index with mixed timezones. If your CSV
 file contains columns with a mixture of timezones, the default result will be
 an object-dtype column with strings, even with ``parse_dates``.
+To parse the mixed-timezone values as a datetime column, read in as ``object`` dtype and
+then call :func:`to_datetime` with ``utc=True``.
 
 
 .. ipython:: python
@@ -939,14 +941,6 @@ an object-dtype column with strings, even with ``parse_dates``.
    a
    2000-01-01T00:00:00+05:00
    2000-01-01T00:00:00+06:00"""
-   df = pd.read_csv(StringIO(content), parse_dates=["a"])
-   df["a"]
-
-To parse the mixed-timezone values as a datetime column, read in as ``object`` dtype and
-then call :func:`to_datetime` with ``utc=True``.
-
-.. ipython:: python
-
    df = pd.read_csv(StringIO(content))
    df["a"] = pd.to_datetime(df["a"], utc=True)
    df["a"]