Merge branch 'main' into groupby-then-resample-if-index-out-of-order

Author: aram-cedarwood

.circleci/config.yml

@@ -56,7 +56,7 @@ jobs:
             /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
             . ~/virtualenvs/pandas-dev/bin/activate
             python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
             python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
             python -m pip list --no-cache-dir
             export PANDAS_CI=1

.github/workflows/unit-tests.yml

@@ -236,7 +236,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -274,7 +274,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
 
@@ -295,7 +295,7 @@ jobs:
     # In general, this will remain frozen(present, but not running) until:
     #    - The next unreleased Python version has released beta 1
     #      - This version should be available on GitHub Actions.
-    #    - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil)
+    #    - Our required build/runtime dependencies(numpy, Cython, python-dateutil)
     #      support that unreleased Python version.
     #    To unfreeze, comment out the ``if: false`` condition, and make sure you update
     #    the name of the workflow and Python version in actions/setup-python ``python-version:``
@@ -348,7 +348,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
+          python -m pip install python-dateutil tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
           python -m pip list
 

ci/code_checks.sh

@@ -158,26 +158,15 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.sparse.sp_values SA01" \
         -i "pandas.Series.sparse.to_coo PR07,RT03,SA01" \
         -i "pandas.Series.std PR01,RT03,SA01" \
-        -i "pandas.Series.str.capitalize RT03" \
-        -i "pandas.Series.str.casefold RT03" \
-        -i "pandas.Series.str.center RT03,SA01" \
-        -i "pandas.Series.str.decode PR07,RT03,SA01" \
-        -i "pandas.Series.str.encode PR07,RT03,SA01" \
-        -i "pandas.Series.str.ljust RT03,SA01" \
-        -i "pandas.Series.str.lower RT03" \
         -i "pandas.Series.str.lstrip RT03" \
         -i "pandas.Series.str.match RT03" \
         -i "pandas.Series.str.normalize RT03,SA01" \
         -i "pandas.Series.str.partition RT03" \
         -i "pandas.Series.str.repeat SA01" \
         -i "pandas.Series.str.replace SA01" \
-        -i "pandas.Series.str.rjust RT03,SA01" \
         -i "pandas.Series.str.rpartition RT03" \
         -i "pandas.Series.str.rstrip RT03" \
         -i "pandas.Series.str.strip RT03" \
-        -i "pandas.Series.str.swapcase RT03" \
-        -i "pandas.Series.str.title RT03" \
-        -i "pandas.Series.str.upper RT03" \
         -i "pandas.Series.str.wrap RT03,SA01" \
         -i "pandas.Series.str.zfill RT03" \
         -i "pandas.Series.struct.dtypes SA01" \
@@ -231,11 +220,8 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Timestamp.to_period PR01,SA01" \
         -i "pandas.Timestamp.today SA01" \
         -i "pandas.Timestamp.toordinal SA01" \
-        -i "pandas.Timestamp.tz_localize SA01" \
         -i "pandas.Timestamp.tzinfo GL08" \
-        -i "pandas.Timestamp.tzname SA01" \
         -i "pandas.Timestamp.unit SA01" \
-        -i "pandas.Timestamp.utcfromtimestamp PR01,SA01" \
         -i "pandas.Timestamp.utcoffset SA01" \
         -i "pandas.Timestamp.utctimetuple SA01" \
         -i "pandas.Timestamp.value GL08" \

ci/deps/actions-310-minimum_versions.yaml

@@ -23,7 +23,6 @@ dependencies:
   # required dependencies
   - python-dateutil=2.8.2
   - numpy=1.23.5
-  - pytz=2020.1
 
   # optional dependencies
   - beautifulsoup4=4.11.2
@@ -49,6 +48,7 @@ dependencies:
   - pyreadstat=1.2.0
   - pytables=3.8.0
   - python-calamine=0.1.7
+  - pytz=2023.4
   - pyxlsb=1.0.10
   - s3fs=2022.11.0
   - scipy=1.10.0

ci/deps/actions-310.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-311-downstream_compat.yaml

@@ -22,7 +22,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -48,6 +47,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-311-numpydev.yaml

@@ -18,7 +18,6 @@ dependencies:
 
   # pandas dependencies
   - python-dateutil
-  - pytz
   - pip
 
   - pip:

ci/deps/actions-311-pyarrownightly.yaml

@@ -19,7 +19,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy<2
-  - pytz
   - pip
 
   - pip:

ci/deps/actions-311.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-312.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-pypy-39.yaml

@@ -22,6 +22,5 @@ dependencies:
   # required
   - numpy
   - python-dateutil
-  - pytz
   - pip:
     - tzdata>=2022.7

ci/deps/circle-311-arm64.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/meta.yaml

@@ -37,7 +37,6 @@ requirements:
     - numpy >=1.21.6  # [py<311]
     - numpy >=1.23.2  # [py>=311]
     - python-dateutil >=2.8.2
-    - pytz >=2020.1
     - python-tzdata >=2022.7
 
 test:

doc/source/development/contributing_docstring.rst

@@ -142,7 +142,7 @@ backticks. The following are considered inline code:
 
         With several mistakes in the docstring.
 
-        It has a blank like after the signature ``def func():``.
+        It has a blank line after the signature ``def func():``.
 
         The text 'Some function' should go in the line after the
         opening quotes of the docstring, not in the same line.

doc/source/getting_started/install.rst

@@ -205,7 +205,6 @@ Package                                                          Minimum support
 ================================================================ ==========================
 `NumPy <https://numpy.org>`__                                    1.23.5
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.8.2
-`pytz <https://pypi.org/project/pytz/>`__                        2020.1
 `tzdata <https://pypi.org/project/tzdata/>`__                    2022.7
 ================================================================ ==========================
 
@@ -419,3 +418,14 @@ Dependency                Minimum Version    pip extra       Notes
 ========================= ================== =============== =============================================================
 Zstandard                 0.19.0             compression     Zstandard compression
 ========================= ================== =============== =============================================================
+
+Timezone
+^^^^^^^^
+
+Installable with ``pip install "pandas[timezone]"``
+
+========================= ================== =================== =============================================================
+Dependency                Minimum Version    pip extra           Notes
+========================= ================== =================== =============================================================
+pytz                      2023.4             timezone            Alternative timezone library to ``zoneinfo``.
+========================= ================== =================== =============================================================

doc/source/user_guide/timeseries.rst

@@ -2569,7 +2569,7 @@ Ambiguous times when localizing
 because daylight savings time (DST) in a local time zone causes some times to occur
 twice within one day ("clocks fall back"). The following options are available:
 
-* ``'raise'``: Raises a ``pytz.AmbiguousTimeError`` (the default behavior)
+* ``'raise'``: Raises a ``ValueError`` (the default behavior)
 * ``'infer'``: Attempt to determine the correct offset base on the monotonicity of the timestamps
 * ``'NaT'``: Replaces ambiguous times with ``NaT``
 * ``bool``: ``True`` represents a DST time, ``False`` represents non-DST time. An array-like of ``bool`` values is supported for a sequence of times.
@@ -2604,7 +2604,7 @@ A DST transition may also shift the local time ahead by 1 hour creating nonexist
 local times ("clocks spring forward"). The behavior of localizing a timeseries with nonexistent times
 can be controlled by the ``nonexistent`` argument. The following options are available:
 
-* ``'raise'``: Raises a ``pytz.NonExistentTimeError`` (the default behavior)
+* ``'raise'``: Raises a ``ValueError`` (the default behavior)
 * ``'NaT'``: Replaces nonexistent times with ``NaT``
 * ``'shift_forward'``: Shifts nonexistent times forward to the closest real time
 * ``'shift_backward'``: Shifts nonexistent times backward to the closest real time

doc/source/whatsnew/v3.0.0.rst

@@ -50,6 +50,7 @@ Other enhancements
 - :meth:`DataFrame.pivot_table` and :func:`pivot_table` now allow the passing of keyword arguments to ``aggfunc`` through ``**kwargs`` (:issue:`57884`)
 - :meth:`Series.cummin` and :meth:`Series.cummax` now supports :class:`CategoricalDtype` (:issue:`52335`)
 - :meth:`Series.plot` now correctly handle the ``ylabel`` parameter for pie charts, allowing for explicit control over the y-axis label (:issue:`58239`)
+- :meth:`pandas.concat` will raise a ``ValueError`` when ``ignore_index=True`` and ``keys`` is not ``None`` (:issue:`59274`)
 - Multiplying two :class:`DateOffset` objects will now raise a ``TypeError`` instead of a ``RecursionError`` (:issue:`59442`)
 - Restore support for reading Stata 104-format and enable reading 103-format dta files (:issue:`58554`)
 - Support reading Stata 102-format (Stata 1) dta files (:issue:`58978`)
@@ -221,6 +222,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +------------------------+---------------------+
 | Package                | New Minimum Version |
 +========================+=====================+
+| pytz                   | 2023.4              |
++------------------------+---------------------+
 | fastparquet            | 2023.10.0           |
 +------------------------+---------------------+
 | adbc-driver-postgresql | 0.10.0              |
@@ -230,6 +233,37 @@ Optional libraries below the lowest tested version may still work, but are not c
 
 See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
+.. _whatsnew_300.api_breaking.pytz:
+
+``pytz`` now an optional dependency
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+pandas now uses :py:mod:`zoneinfo` from the standard library as the default timezone implementation when passing a timezone
+string to various methods. (:issue:`34916`)
+
+*Old behavior:*
+
+.. code-block:: ipython
+
+    In [1]: ts = pd.Timestamp(2024, 1, 1).tz_localize("US/Pacific")
+    In [2]: ts.tz
+    <DstTzInfo 'US/Pacific' LMT-1 day, 16:07:00 STD>
+
+*New behavior:*
+
+.. ipython:: python
+
+    ts = pd.Timestamp(2024, 1, 1).tz_localize("US/Pacific")
+    ts.tz
+
+``pytz`` timezone objects are still supported when passed directly, but they will no longer be returned by default
+from string inputs. Moreover, ``pytz`` is no longer a required dependency of pandas, but can be installed
+with the pip extra ``pip install pandas[timezone]``.
+
+
+Additionally, pandas no longer throws ``pytz`` exceptions for timezone operations leading to ambiguous or nonexistent
+times. These cases will now raise a ``ValueError``.
+
 .. _whatsnew_300.api_breaking.other:
 
 Other API changes
@@ -621,6 +655,7 @@ Reshaping
 ^^^^^^^^^
 - Bug in :func:`qcut` where values at the quantile boundaries could be incorrectly assigned (:issue:`59355`)
 - Bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
+- Bug in :meth:`DataFrame.merge` where merging on a column containing only ``NaN`` values resulted in an out-of-bounds array access (:issue:`59421`)
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when ``sort=False`` (:issue:`54987`, :issue:`55516`)
 - Bug in :meth:`DataFrame.unstack` producing incorrect results when manipulating empty :class:`DataFrame` with an :class:`ExtentionDtype` (:issue:`59123`)
 

environment.yml

@@ -24,7 +24,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy<2
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -50,6 +49,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

pandas/__init__.py

@@ -3,7 +3,7 @@
 __docformat__ = "restructuredtext"
 
 # Let users know if they're missing any of our hard dependencies
-_hard_dependencies = ("numpy", "pytz", "dateutil")
+_hard_dependencies = ("numpy", "dateutil")
 _missing_dependencies = []
 
 for _dependency in _hard_dependencies:

pandas/_config/config.py

@@ -426,6 +426,11 @@ def option_context(*args) -> Generator[None, None, None]:
     None
         No return value.
 
+    Yields
+    ------
+    None
+        No yield value.
+
     See Also
     --------
     get_option : Retrieve the value of the specified option.

pandas/_libs/tslibs/conversion.pyx

@@ -69,6 +69,7 @@ from pandas._libs.tslibs.timestamps cimport _Timestamp
 from pandas._libs.tslibs.timezones cimport (
     get_utcoffset,
     is_utc,
+    treat_tz_as_pytz,
 )
 from pandas._libs.tslibs.tzconversion cimport (
     Localizer,
@@ -747,11 +748,17 @@ cdef datetime _localize_pydatetime(datetime dt, tzinfo tz):
         identically, i.e. discards nanos from Timestamps.
         It also assumes that the `tz` input is not None.
     """
-    try:
+    if treat_tz_as_pytz(tz):
+        import pytz
+
         # datetime.replace with pytz may be incorrect result
         # TODO: try to respect `fold` attribute
-        return tz.localize(dt, is_dst=None)
-    except AttributeError:
+        try:
+            return tz.localize(dt, is_dst=None)
+        except (pytz.AmbiguousTimeError, pytz.NonExistentTimeError) as err:
+            # As of pandas 3.0, we raise ValueErrors instead of pytz exceptions
+            raise ValueError(str(err)) from err
+    else:
         return dt.replace(tzinfo=tz)