CI: start testing Python 3.11 (#47442)

* CI: start testing Python 3.11

* fix

* fix

* fix macos

* add error msg compat

* fix error msg in test

* xfail and skip tests

* fix error

* roll back

* revert

* maybe fix errors

* update

* try something

* pin hypothesis

* Don't check parentheses equality

* typo

* pin cython

* correctly pin hypothesis

* adjust test

* remove compiler flag

* typo'ed

* fix tests

* linter

* clarify freeze/unfreeze instructions

* whitespace

* one more

Co-authored-by: Thomas Li <[email protected]>

Author: fangchenli

.github/workflows/python-dev.yml

@@ -1,9 +1,21 @@
-# This file is purposely frozen(does not run). DO NOT DELETE IT
-# Unfreeze(by commentingthe if: false() condition) once the
-# next Python Dev version has released beta 1 and both Cython and numpy support it
-# After that Python has released, migrate the workflows to the
-# posix GHA workflows and "freeze" this file by
-# uncommenting the if: false() condition
+# This workflow may or may not run depending on the state of the next
+# unreleased Python version. DO NOT DELETE IT.
+#
+# In general, this file will remain frozen(present, but not running) until:
+#    - The next unreleased Python version has released beta 1
+#      - This version should be available on Github Actions.
+#    - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil)
+#      support that unreleased Python version.
+#    To unfreeze, comment out the ``if: false`` condition, and make sure you update
+#    the name of the workflow and Python version in actions/setup-python to: '3.12-dev'
+#
+# After it has been unfrozen, this file should remain unfrozen(present, and running) until:
+#    - The next Python version has been officially released.
+#    OR
+#    - Most/All of our optional dependencies support Python 3.11 AND
+#    - The next Python version has released a rc(we are guaranteed a stable ABI).
+#    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
+#    to the corresponding posix/windows-macos/sdist etc. workflows.
 # Feel free to modify this comment as necessary.
 
 name: Python Dev
@@ -32,7 +44,7 @@ permissions:
 
 jobs:
   build:
-    if: false # Comment this line out to "unfreeze"
+    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -53,27 +65,27 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python Dev Version
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: '3.11-dev'
 
     - name: Install dependencies
-      shell: bash -el {0}
       run: |
-        python3 -m pip install --upgrade pip setuptools wheel
-        python3 -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
-        python3 -m pip install git+https://github.com/nedbat/coveragepy.git
-        python3 -m pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
-        python3 -m pip list
+        python --version
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install git+https://github.com/numpy/numpy.git
+        python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
+        python -m pip list
 
     - name: Build Pandas
       run: |
-        python3 setup.py build_ext -q -j2
-        python3 -m pip install -e . --no-build-isolation --no-use-pep517
+        python setup.py build_ext -q -j2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
 
     - name: Build Version
       run: |
-        python3 -c "import pandas; pandas.show_versions();"
+        python -c "import pandas; pandas.show_versions();"
 
     - name: Test
       uses: ./.github/actions/run-tests

pandas/compat/__init__.py

@@ -36,6 +36,7 @@
 
 PY39 = sys.version_info >= (3, 9)
 PY310 = sys.version_info >= (3, 10)
+PY311 = sys.version_info >= (3, 11)
 PYPY = platform.python_implementation() == "PyPy"
 IS64 = sys.maxsize > 2**32
 

pandas/tests/arrays/categorical/test_api.py

@@ -3,6 +3,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat import PY311
+
 from pandas import (
     Categorical,
     CategoricalIndex,
@@ -61,7 +63,11 @@ def test_set_ordered(self):
         assert not cat2.ordered
 
         # removed in 0.19.0
-        msg = "can't set attribute"
+        msg = (
+            "property 'ordered' of 'Categorical' object has no setter"
+            if PY311
+            else "can't set attribute"
+        )
         with pytest.raises(AttributeError, match=msg):
             cat.ordered = True
         with pytest.raises(AttributeError, match=msg):
@@ -515,7 +521,12 @@ def test_codes_immutable(self):
         tm.assert_numpy_array_equal(c.codes, exp)
 
         # Assignments to codes should raise
-        with pytest.raises(AttributeError, match="can't set attribute"):
+        msg = (
+            "property 'codes' of 'Categorical' object has no setter"
+            if PY311
+            else "can't set attribute"
+        )
+        with pytest.raises(AttributeError, match=msg):
             c.codes = np.array([0, 1, 2, 0, 1], dtype="int8")
 
         # changes in the codes array should raise

pandas/tests/indexes/datetimes/test_constructors.py

@@ -1137,7 +1137,10 @@ def test_timestamp_constructor_retain_fold(tz, fold):
 
 _tzs = ["dateutil/Europe/London"]
 if PY39:
-    _tzs = ["dateutil/Europe/London", zoneinfo.ZoneInfo("Europe/London")]
+    try:
+        _tzs = ["dateutil/Europe/London", zoneinfo.ZoneInfo("Europe/London")]
+    except zoneinfo.ZoneInfoNotFoundError:
+        pass
 
 
 @pytest.mark.parametrize("tz", _tzs)

pandas/tests/indexes/multi/test_get_set.py

@@ -1,6 +1,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat import PY311
+
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
 
 import pandas as pd
@@ -139,9 +141,15 @@ def test_set_levels_codes_directly(idx):
     minor_codes = [(x + 1) % 1 for x in minor_codes]
     new_codes = [major_codes, minor_codes]
 
-    msg = "[Cc]an't set attribute"
+    msg = "Can't set attribute"
     with pytest.raises(AttributeError, match=msg):
         idx.levels = new_levels
+
+    msg = (
+        "property 'codes' of 'MultiIndex' object has no setter"
+        if PY311
+        else "can't set attribute"
+    )
     with pytest.raises(AttributeError, match=msg):
         idx.codes = new_codes
 

pandas/tests/indexes/period/test_freq_attr.py

@@ -1,5 +1,7 @@
 import pytest
 
+from pandas.compat import PY311
+
 from pandas import (
     offsets,
     period_range,
@@ -17,5 +19,10 @@ def test_freq_setter_deprecated(self):
             idx.freq
 
         # warning for setter
-        with pytest.raises(AttributeError, match="can't set attribute"):
+        msg = (
+            "property 'freq' of 'PeriodArray' object has no setter"
+            if PY311
+            else "can't set attribute"
+        )
+        with pytest.raises(AttributeError, match=msg):
             idx.freq = offsets.Day()

pandas/tests/io/parser/common/test_read_errors.py

@@ -12,6 +12,7 @@
 import numpy as np
 import pytest
 
+from pandas.compat import PY311
 from pandas.errors import (
     EmptyDataError,
     ParserError,
@@ -224,13 +225,19 @@ def test_read_csv_wrong_num_columns(all_parsers):
         parser.read_csv(StringIO(data))
 
 
-def test_null_byte_char(all_parsers):
+def test_null_byte_char(request, all_parsers):
     # see gh-2741
     data = "\x00,foo"
     names = ["a", "b"]
     parser = all_parsers
 
-    if parser.engine == "c":
+    if parser.engine == "c" or (parser.engine == "python" and PY311):
+        if parser.engine == "python" and PY311:
+            request.node.add_marker(
+                pytest.mark.xfail(
+                    reason="In Python 3.11, this is read as an empty character not null"
+                )
+            )
         expected = DataFrame([[np.nan, "foo"]], columns=names)
         out = parser.read_csv(StringIO(data), names=names)
         tm.assert_frame_equal(out, expected)

pandas/tests/io/parser/test_quoting.py

@@ -8,6 +8,7 @@
 
 import pytest
 
+from pandas.compat import PY311
 from pandas.errors import ParserError
 
 from pandas import DataFrame
@@ -80,11 +81,16 @@ def test_null_quote_char(all_parsers, quoting, quote_char):
 
     if quoting != csv.QUOTE_NONE:
         # Sanity checking.
-        msg = "quotechar must be set if quoting enabled"
+        msg = (
+            '"quotechar" must be a 1-character string'
+            if PY311 and all_parsers.engine == "python" and quote_char == ""
+            else "quotechar must be set if quoting enabled"
+        )
 
         with pytest.raises(TypeError, match=msg):
             parser.read_csv(StringIO(data), **kwargs)
-    else:
+    elif not (PY311 and all_parsers.engine == "python"):
+        # Python 3.11+ doesn't support null/blank quote chars in their csv parsers
         expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"])
         result = parser.read_csv(StringIO(data), **kwargs)
         tm.assert_frame_equal(result, expected)