DEPS: Make pytz an optional dependency (pandas-dev#59089)

* Make pytz an optional dependency

* Start to address tests

* Fix tests

* Fix tests

* Fix test, import optional pytz in conftest

* Fix formatting

* Change minimum

* remove type ignore

* another pa under 17

* Address comments

* Undo file

* Fix pyarrow 17 test

* Test xpasses on pyarrow 18

Author: mroeschke

.circleci/config.yml

@@ -56,7 +56,7 @@ jobs:
             /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
             . ~/virtualenvs/pandas-dev/bin/activate
             python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+            python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
             python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
             python -m pip list --no-cache-dir
             export PANDAS_CI=1

.github/workflows/unit-tests.yml

@@ -236,7 +236,7 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy --config-settings=setup-args="-Dallow-noblas=true"
-          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+          python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -274,7 +274,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=7.3.2 pytest-xdist>=3.4.0 hypothesis>=6.84.0
           python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
           python -m pip list --no-cache-dir
 
@@ -295,7 +295,7 @@ jobs:
     # In general, this will remain frozen(present, but not running) until:
     #    - The next unreleased Python version has released beta 1
     #      - This version should be available on GitHub Actions.
-    #    - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil)
+    #    - Our required build/runtime dependencies(numpy, Cython, python-dateutil)
     #      support that unreleased Python version.
     #    To unfreeze, comment out the ``if: false`` condition, and make sure you update
     #    the name of the workflow and Python version in actions/setup-python ``python-version:``
@@ -348,7 +348,7 @@ jobs:
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
+          python -m pip install python-dateutil tzdata cython hypothesis>=6.84.0 pytest>=7.3.2 pytest-xdist>=3.4.0 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps --config-settings=setup-args="--werror"
           python -m pip list
 

ci/deps/actions-310-minimum_versions.yaml

@@ -23,7 +23,6 @@ dependencies:
   # required dependencies
   - python-dateutil=2.8.2
   - numpy=1.23.5
-  - pytz=2020.1
 
   # optional dependencies
   - beautifulsoup4=4.11.2
@@ -49,6 +48,7 @@ dependencies:
   - pyreadstat=1.2.0
   - pytables=3.8.0
   - python-calamine=0.1.7
+  - pytz=2023.4
   - pyxlsb=1.0.10
   - s3fs=2022.11.0
   - scipy=1.10.0

ci/deps/actions-310.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-311-downstream_compat.yaml

@@ -22,7 +22,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -48,6 +47,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-311-numpydev.yaml

@@ -18,7 +18,6 @@ dependencies:
 
   # pandas dependencies
   - python-dateutil
-  - pytz
   - pip
 
   - pip:

ci/deps/actions-311-pyarrownightly.yaml

@@ -19,7 +19,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy<2
-  - pytz
   - pip
 
   - pip:

ci/deps/actions-311.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-312.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/deps/actions-pypy-39.yaml

@@ -22,6 +22,5 @@ dependencies:
   # required
   - numpy
   - python-dateutil
-  - pytz
   - pip:
     - tzdata>=2022.7

ci/deps/circle-311-arm64.yaml

@@ -21,7 +21,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -47,6 +46,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

ci/meta.yaml

@@ -37,7 +37,6 @@ requirements:
     - numpy >=1.21.6  # [py<311]
     - numpy >=1.23.2  # [py>=311]
     - python-dateutil >=2.8.2
-    - pytz >=2020.1
     - python-tzdata >=2022.7
 
 test:

doc/source/getting_started/install.rst

@@ -205,7 +205,6 @@ Package                                                          Minimum support
 ================================================================ ==========================
 `NumPy <https://numpy.org>`__                                    1.23.5
 `python-dateutil <https://dateutil.readthedocs.io/en/stable/>`__ 2.8.2
-`pytz <https://pypi.org/project/pytz/>`__                        2020.1
 `tzdata <https://pypi.org/project/tzdata/>`__                    2022.7
 ================================================================ ==========================
 
@@ -419,3 +418,14 @@ Dependency                Minimum Version    pip extra       Notes
 ========================= ================== =============== =============================================================
 Zstandard                 0.19.0             compression     Zstandard compression
 ========================= ================== =============== =============================================================
+
+Timezone
+^^^^^^^^
+
+Installable with ``pip install "pandas[timezone]"``
+
+========================= ================== =================== =============================================================
+Dependency                Minimum Version    pip extra           Notes
+========================= ================== =================== =============================================================
+pytz                      2023.4             timezone            Alternative timezone library to ``zoneinfo``.
+========================= ================== =================== =============================================================

doc/source/user_guide/timeseries.rst

@@ -2569,7 +2569,7 @@ Ambiguous times when localizing
 because daylight savings time (DST) in a local time zone causes some times to occur
 twice within one day ("clocks fall back"). The following options are available:
 
-* ``'raise'``: Raises a ``pytz.AmbiguousTimeError`` (the default behavior)
+* ``'raise'``: Raises a ``ValueError`` (the default behavior)
 * ``'infer'``: Attempt to determine the correct offset base on the monotonicity of the timestamps
 * ``'NaT'``: Replaces ambiguous times with ``NaT``
 * ``bool``: ``True`` represents a DST time, ``False`` represents non-DST time. An array-like of ``bool`` values is supported for a sequence of times.
@@ -2604,7 +2604,7 @@ A DST transition may also shift the local time ahead by 1 hour creating nonexist
 local times ("clocks spring forward"). The behavior of localizing a timeseries with nonexistent times
 can be controlled by the ``nonexistent`` argument. The following options are available:
 
-* ``'raise'``: Raises a ``pytz.NonExistentTimeError`` (the default behavior)
+* ``'raise'``: Raises a ``ValueError`` (the default behavior)
 * ``'NaT'``: Replaces nonexistent times with ``NaT``
 * ``'shift_forward'``: Shifts nonexistent times forward to the closest real time
 * ``'shift_backward'``: Shifts nonexistent times backward to the closest real time

doc/source/whatsnew/v3.0.0.rst

@@ -221,6 +221,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +------------------------+---------------------+
 | Package                | New Minimum Version |
 +========================+=====================+
+| pytz                   | 2023.4              |
++------------------------+---------------------+
 | fastparquet            | 2023.10.0           |
 +------------------------+---------------------+
 | adbc-driver-postgresql | 0.10.0              |
@@ -230,6 +232,37 @@ Optional libraries below the lowest tested version may still work, but are not c
 
 See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
+.. _whatsnew_300.api_breaking.pytz:
+
+``pytz`` now an optional dependency
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+pandas now uses :py:mod:`zoneinfo` from the standard library as the default timezone implementation when passing a timezone
+string to various methods. (:issue:`34916`)
+
+*Old behavior:*
+
+.. code-block:: ipython
+
+    In [1]: ts = pd.Timestamp(2024, 1, 1).tz_localize("US/Pacific")
+    In [2]: ts.tz
+    <DstTzInfo 'US/Pacific' LMT-1 day, 16:07:00 STD>
+
+*New behavior:*
+
+.. ipython:: python
+
+    ts = pd.Timestamp(2024, 1, 1).tz_localize("US/Pacific")
+    ts.tz
+
+``pytz`` timezone objects are still supported when passed directly, but they will no longer be returned by default
+from string inputs. Moreover, ``pytz`` is no longer a required dependency of pandas, but can be installed
+with the pip extra ``pip install pandas[timezone]``.
+
+
+Additionally, pandas no longer throws ``pytz`` exceptions for timezone operations leading to ambiguous or nonexistent
+times. These cases will now raise a ``ValueError``.
+
 .. _whatsnew_300.api_breaking.other:
 
 Other API changes

environment.yml

@@ -24,7 +24,6 @@ dependencies:
   # required dependencies
   - python-dateutil
   - numpy<2
-  - pytz
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
@@ -50,6 +49,7 @@ dependencies:
   - pyreadstat>=1.2.0
   - pytables>=3.8.0
   - python-calamine>=0.1.7
+  - pytz>=2023.4
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
   - scipy>=1.10.0

pandas/__init__.py

@@ -3,7 +3,7 @@
 __docformat__ = "restructuredtext"
 
 # Let users know if they're missing any of our hard dependencies
-_hard_dependencies = ("numpy", "pytz", "dateutil")
+_hard_dependencies = ("numpy", "dateutil")
 _missing_dependencies = []
 
 for _dependency in _hard_dependencies:

pandas/_libs/tslibs/conversion.pyx

@@ -69,6 +69,7 @@ from pandas._libs.tslibs.timestamps cimport _Timestamp
 from pandas._libs.tslibs.timezones cimport (
     get_utcoffset,
     is_utc,
+    treat_tz_as_pytz,
 )
 from pandas._libs.tslibs.tzconversion cimport (
     Localizer,
@@ -747,11 +748,17 @@ cdef datetime _localize_pydatetime(datetime dt, tzinfo tz):
         identically, i.e. discards nanos from Timestamps.
         It also assumes that the `tz` input is not None.
     """
-    try:
+    if treat_tz_as_pytz(tz):
+        import pytz
+
         # datetime.replace with pytz may be incorrect result
         # TODO: try to respect `fold` attribute
-        return tz.localize(dt, is_dst=None)
-    except AttributeError:
+        try:
+            return tz.localize(dt, is_dst=None)
+        except (pytz.AmbiguousTimeError, pytz.NonExistentTimeError) as err:
+            # As of pandas 3.0, we raise ValueErrors instead of pytz exceptions
+            raise ValueError(str(err)) from err
+    else:
         return dt.replace(tzinfo=tz)
 
 

pandas/_libs/tslibs/nattype.pyx

@@ -1067,7 +1067,7 @@ class NaTType(_NaT):
             * bool contains flags to determine if time is dst or not (note
               that this flag is only applicable for ambiguous fall dst dates).
             * 'NaT' will return NaT for an ambiguous time.
-            * 'raise' will raise an AmbiguousTimeError for an ambiguous time.
+            * 'raise' will raise a ValueError for an ambiguous time.
 
         nonexistent : {'raise', 'shift_forward', 'shift_backward', 'NaT', \
 timedelta}, default 'raise'
@@ -1080,7 +1080,7 @@ timedelta}, default 'raise'
               closest existing time.
             * 'NaT' will return NaT where there are nonexistent times.
             * timedelta objects will shift nonexistent times by the timedelta.
-            * 'raise' will raise an NonExistentTimeError if there are
+            * 'raise' will raise a ValueError if there are
               nonexistent times.
 
         Returns
@@ -1168,7 +1168,7 @@ timedelta}, default 'raise'
             * bool contains flags to determine if time is dst or not (note
               that this flag is only applicable for ambiguous fall dst dates).
             * 'NaT' will return NaT for an ambiguous time.
-            * 'raise' will raise an AmbiguousTimeError for an ambiguous time.
+            * 'raise' will raise a ValueError for an ambiguous time.
 
         nonexistent : {'raise', 'shift_forward', 'shift_backward', 'NaT', \
 timedelta}, default 'raise'
@@ -1181,7 +1181,7 @@ timedelta}, default 'raise'
               closest existing time.
             * 'NaT' will return NaT where there are nonexistent times.
             * timedelta objects will shift nonexistent times by the timedelta.
-            * 'raise' will raise an NonExistentTimeError if there are
+            * 'raise' will raise a ValueError if there are
               nonexistent times.
 
         Raises
@@ -1263,7 +1263,7 @@ timedelta}, default 'raise'
             * bool contains flags to determine if time is dst or not (note
               that this flag is only applicable for ambiguous fall dst dates).
             * 'NaT' will return NaT for an ambiguous time.
-            * 'raise' will raise an AmbiguousTimeError for an ambiguous time.
+            * 'raise' will raise a ValueError for an ambiguous time.
 
         nonexistent : {'raise', 'shift_forward', 'shift_backward', 'NaT', \
 timedelta}, default 'raise'
@@ -1276,7 +1276,7 @@ timedelta}, default 'raise'
               closest existing time.
             * 'NaT' will return NaT where there are nonexistent times.
             * timedelta objects will shift nonexistent times by the timedelta.
-            * 'raise' will raise an NonExistentTimeError if there are
+            * 'raise' will raise a ValueError if there are
               nonexistent times.
 
         Raises
@@ -1427,7 +1427,7 @@ timedelta}, default 'raise'
             * bool contains flags to determine if time is dst or not (note
               that this flag is only applicable for ambiguous fall dst dates).
             * 'NaT' will return NaT for an ambiguous time.
-            * 'raise' will raise an AmbiguousTimeError for an ambiguous time.
+            * 'raise' will raise a ValueError for an ambiguous time.
 
         nonexistent : 'shift_forward', 'shift_backward', 'NaT', timedelta, \
 default 'raise'
@@ -1442,7 +1442,7 @@ default 'raise'
               closest existing time.
             * 'NaT' will return NaT where there are nonexistent times.
             * timedelta objects will shift nonexistent times by the timedelta.
-            * 'raise' will raise an NonExistentTimeError if there are
+            * 'raise' will raise a ValueError if there are
               nonexistent times.
 
         Returns