From 5fbeaebb8a34567498cb750c93c8341930a7c5d1 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 20 Jun 2022 22:55:42 -0700 Subject: [PATCH 01/26] CI: start testing Python 3.11 --- .github/workflows/python-dev.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 09639acafbba1..d58a844f36908 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -1,11 +1,3 @@ -# This file is purposely frozen(does not run). DO NOT DELETE IT -# Unfreeze(by commentingthe if: false() condition) once the -# next Python Dev version has released beta 1 and both Cython and numpy support it -# After that Python has released, migrate the workflows to the -# posix GHA workflows and "freeze" this file by -# uncommenting the if: false() condition -# Feel free to modify this comment as necessary. - name: Python Dev on: @@ -29,7 +21,6 @@ env: jobs: build: - if: false # Comment this line out to "unfreeze" runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -50,7 +41,7 @@ jobs: fetch-depth: 0 - name: Set up Python Dev Version - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: '3.11-dev' @@ -58,9 +49,10 @@ jobs: shell: bash -el {0} run: | python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install git+https://github.com/cython/cython.git python3 -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy python3 -m pip install git+https://github.com/nedbat/coveragepy.git - python3 -m pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python3 -m pip install python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python3 -m pip list - name: Build Pandas From 4805a6b8dd7c5ed969a9f21f13edb1a1919f2c49 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 20 Jun 2022 23:06:23 -0700 Subject: [PATCH 02/26] fix --- .github/workflows/python-dev.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index d58a844f36908..a1ed7a1156eb3 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -48,21 +48,22 @@ jobs: - name: Install dependencies shell: bash -el {0} run: | - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip install git+https://github.com/cython/cython.git - python3 -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - python3 -m pip install git+https://github.com/nedbat/coveragepy.git - python3 -m pip install python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 - python3 -m pip list + python --version + python -m pip install --upgrade pip setuptools wheel + python -m pip install git+https://github.com/cython/cython.git + python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + python -m pip install git+https://github.com/nedbat/coveragepy.git + python -m pip install python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip list - name: Build Pandas run: | - python3 setup.py build_ext -q -j2 - python3 -m pip install -e . --no-build-isolation --no-use-pep517 + python setup.py build_ext -q -j2 + python -m pip install -e . --no-build-isolation --no-use-pep517 - name: Build Version run: | - python3 -c "import pandas; pandas.show_versions();" + python -c "import pandas; pandas.show_versions();" - name: Test uses: ./.github/actions/run-tests From 9f038f3ad0f184d5f088e299ad32cc9a1f7ccf61 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 20 Jun 2022 23:38:04 -0700 Subject: [PATCH 03/26] fix --- .github/workflows/python-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a1ed7a1156eb3..b39d9ab6b7875 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -51,7 +51,7 @@ jobs: python --version python -m pip install --upgrade pip setuptools wheel python -m pip install git+https://github.com/cython/cython.git - python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + python -m pip install git+https://github.com/numpy/numpy.git python -m pip install git+https://github.com/nedbat/coveragepy.git python -m pip install python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list From 2b45e3280e43017de2a712ce11f3b1156da2f904 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jun 2022 10:52:12 -0700 Subject: [PATCH 04/26] fix macos --- .github/workflows/python-dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index b39d9ab6b7875..d2a8dd7e115de 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -46,7 +46,6 @@ jobs: python-version: '3.11-dev' - name: Install dependencies - shell: bash -el {0} run: | python --version python -m pip install --upgrade pip setuptools wheel From bfbd17bc19dba30574cf6ff60c73d15beaaf105e Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jun 2022 14:03:41 -0700 Subject: [PATCH 05/26] add error msg compat --- pandas/compat/__init__.py | 1 + pandas/tests/arrays/categorical/test_api.py | 15 +++++++++++++-- pandas/tests/indexes/multi/test_get_set.py | 14 +++++++++++++- pandas/tests/indexes/period/test_freq_attr.py | 9 ++++++++- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py index bb4787f07b2f0..5d3a36a902c05 100644 --- a/pandas/compat/__init__.py +++ b/pandas/compat/__init__.py @@ -28,6 +28,7 @@ PY39 = sys.version_info >= (3, 9) PY310 = sys.version_info >= (3, 10) +PY311 = sys.version_info >= (3, 11) PYPY = platform.python_implementation() == "PyPy" IS64 = sys.maxsize > 2**32 diff --git a/pandas/tests/arrays/categorical/test_api.py b/pandas/tests/arrays/categorical/test_api.py index 87b1bb88aeac3..d6ff25a960857 100644 --- a/pandas/tests/arrays/categorical/test_api.py +++ b/pandas/tests/arrays/categorical/test_api.py @@ -3,6 +3,8 @@ import numpy as np import pytest +from pandas.compat import PY311 + from pandas import ( Categorical, CategoricalIndex, @@ -53,7 +55,11 @@ def test_set_ordered(self): assert not cat2.ordered # removed in 0.19.0 - msg = "can't set attribute" + msg = ( + "property 'ordered' of 'Categorical' object has no setter" + if PY311 + else "can't set attribute" + ) with pytest.raises(AttributeError, match=msg): cat.ordered = True with pytest.raises(AttributeError, match=msg): @@ -507,7 +513,12 @@ def test_codes_immutable(self): tm.assert_numpy_array_equal(c.codes, exp) # Assignments to codes should raise - with pytest.raises(AttributeError, match="can't set attribute"): + msg = ( + "property 'codes' of 'Categorical' object has no setter" + if PY311 + else "can't set attribute" + ) + with pytest.raises(AttributeError, match=msg): c.codes = np.array([0, 1, 2, 0, 1], dtype="int8") # changes in the codes array should raise diff --git a/pandas/tests/indexes/multi/test_get_set.py b/pandas/tests/indexes/multi/test_get_set.py index aa0e91cecd4fc..b22eb1f75cf6a 100644 --- a/pandas/tests/indexes/multi/test_get_set.py +++ b/pandas/tests/indexes/multi/test_get_set.py @@ -1,6 +1,8 @@ import numpy as np import pytest +from pandas.compat import PY311 + from pandas.core.dtypes.dtypes import DatetimeTZDtype import pandas as pd @@ -139,9 +141,19 @@ def test_set_levels_codes_directly(idx): minor_codes = [(x + 1) % 1 for x in minor_codes] new_codes = [major_codes, minor_codes] - msg = "[Cc]an't set attribute" + msg = ( + "property 'levels' of 'MultiIndex' object has no setter" + if PY311 + else "can't set attribute" + ) with pytest.raises(AttributeError, match=msg): idx.levels = new_levels + + msg = ( + "property 'codes' of 'MultiIndex' object has no setter" + if PY311 + else "can't set attribute" + ) with pytest.raises(AttributeError, match=msg): idx.codes = new_codes diff --git a/pandas/tests/indexes/period/test_freq_attr.py b/pandas/tests/indexes/period/test_freq_attr.py index 3bf3e700e5e72..e1ecffa4982bd 100644 --- a/pandas/tests/indexes/period/test_freq_attr.py +++ b/pandas/tests/indexes/period/test_freq_attr.py @@ -1,5 +1,7 @@ import pytest +from pandas.compat import PY311 + from pandas import ( offsets, period_range, @@ -17,5 +19,10 @@ def test_freq_setter_deprecated(self): idx.freq # warning for setter - with pytest.raises(AttributeError, match="can't set attribute"): + msg = ( + "property 'freq' of 'PeriodArray' object has no setter" + if PY311 + else "can't set attribute" + ) + with pytest.raises(AttributeError, match=msg): idx.freq = offsets.Day() From bac8cbdda4396f4a4672b2386d63d9c3df534b1b Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Thu, 23 Jun 2022 10:08:06 -0700 Subject: [PATCH 06/26] fix error msg in test --- pandas/tests/indexes/multi/test_get_set.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/tests/indexes/multi/test_get_set.py b/pandas/tests/indexes/multi/test_get_set.py index b22eb1f75cf6a..42cf0168f6599 100644 --- a/pandas/tests/indexes/multi/test_get_set.py +++ b/pandas/tests/indexes/multi/test_get_set.py @@ -141,11 +141,7 @@ def test_set_levels_codes_directly(idx): minor_codes = [(x + 1) % 1 for x in minor_codes] new_codes = [major_codes, minor_codes] - msg = ( - "property 'levels' of 'MultiIndex' object has no setter" - if PY311 - else "can't set attribute" - ) + msg = "Can't set attribute" with pytest.raises(AttributeError, match=msg): idx.levels = new_levels From 572acefe24d1e3dad59439b6ef36f2c231c1d061 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 24 Jun 2022 15:34:54 -0700 Subject: [PATCH 07/26] xfail and skip tests --- pandas/tests/io/parser/common/test_read_errors.py | 3 ++- pandas/tests/io/parser/test_quoting.py | 5 +++++ pandas/tests/scalar/test_nat.py | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/parser/common/test_read_errors.py b/pandas/tests/io/parser/common/test_read_errors.py index 47f1052808e0c..57db65b42464b 100644 --- a/pandas/tests/io/parser/common/test_read_errors.py +++ b/pandas/tests/io/parser/common/test_read_errors.py @@ -12,6 +12,7 @@ import numpy as np import pytest +from pandas.compat import PY311 from pandas.errors import ( EmptyDataError, ParserError, @@ -230,7 +231,7 @@ def test_null_byte_char(all_parsers): names = ["a", "b"] parser = all_parsers - if parser.engine == "c": + if parser.engine == "c" or PY311: expected = DataFrame([[np.nan, "foo"]], columns=names) out = parser.read_csv(StringIO(data), names=names) tm.assert_frame_equal(out, expected) diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index 456dd049d2f4a..dd6b1b415e214 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -8,6 +8,7 @@ import pytest +from pandas.compat import PY311 from pandas.errors import ParserError from pandas import DataFrame @@ -79,6 +80,10 @@ def test_null_quote_char(all_parsers, quoting, quote_char): parser = all_parsers if quoting != csv.QUOTE_NONE: + if quote_char == "" and PY311: + pytest.xfail( + "Python 3.11 raises TypeError: 'quotechar' must be a 1-character string here" + ) # Sanity checking. msg = "quotechar must be set if quoting enabled" diff --git a/pandas/tests/scalar/test_nat.py b/pandas/tests/scalar/test_nat.py index 873103b01f64d..b50e9bc6cf175 100644 --- a/pandas/tests/scalar/test_nat.py +++ b/pandas/tests/scalar/test_nat.py @@ -9,6 +9,7 @@ import pytz from pandas._libs.tslibs import iNaT +from pandas.compat import PY311 from pandas.core.dtypes.common import is_datetime64_any_dtype @@ -314,6 +315,7 @@ def test_overlap_public_nat_methods(klass, expected): assert _get_overlap_public_nat_methods(klass) == expected +@pytest.mark.skipif(PY311, reason="error in Python 3.11") @pytest.mark.parametrize( "compare", ( From c5f2fe38b66df538e9827131205afceb5c11081a Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 24 Jun 2022 18:12:20 -0700 Subject: [PATCH 08/26] fix error --- pandas/tests/io/parser/test_quoting.py | 5 +---- pandas/tests/scalar/test_nat.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index dd6b1b415e214..6b623aebfabda 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -72,6 +72,7 @@ def test_quote_char_various(all_parsers, quote_char): tm.assert_frame_equal(result, expected) +@pytest.mark.xfail(PY311, reason="raises TypeError Python 3.11") @pytest.mark.parametrize("quoting", [csv.QUOTE_MINIMAL, csv.QUOTE_NONE]) @pytest.mark.parametrize("quote_char", ["", None]) def test_null_quote_char(all_parsers, quoting, quote_char): @@ -80,10 +81,6 @@ def test_null_quote_char(all_parsers, quoting, quote_char): parser = all_parsers if quoting != csv.QUOTE_NONE: - if quote_char == "" and PY311: - pytest.xfail( - "Python 3.11 raises TypeError: 'quotechar' must be a 1-character string here" - ) # Sanity checking. msg = "quotechar must be set if quoting enabled" diff --git a/pandas/tests/scalar/test_nat.py b/pandas/tests/scalar/test_nat.py index b50e9bc6cf175..8511a6d120ffc 100644 --- a/pandas/tests/scalar/test_nat.py +++ b/pandas/tests/scalar/test_nat.py @@ -315,7 +315,7 @@ def test_overlap_public_nat_methods(klass, expected): assert _get_overlap_public_nat_methods(klass) == expected -@pytest.mark.skipif(PY311, reason="error in Python 3.11") +@pytest.mark.xfail(PY311, reason="error in Python 3.11", raises=TypeError) @pytest.mark.parametrize( "compare", ( From 69ea5eda740f75047c134eeee588df01b59fd76a Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 24 Jun 2022 21:55:24 -0700 Subject: [PATCH 09/26] roll back --- pandas/tests/io/parser/test_quoting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index 6b623aebfabda..8326a07ac5aee 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -72,7 +72,6 @@ def test_quote_char_various(all_parsers, quote_char): tm.assert_frame_equal(result, expected) -@pytest.mark.xfail(PY311, reason="raises TypeError Python 3.11") @pytest.mark.parametrize("quoting", [csv.QUOTE_MINIMAL, csv.QUOTE_NONE]) @pytest.mark.parametrize("quote_char", ["", None]) def test_null_quote_char(all_parsers, quoting, quote_char): From 267c9f2002289465e0b0b32c6fbc723f45d7a428 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 2 Jul 2022 09:30:01 -0700 Subject: [PATCH 10/26] revert --- pandas/tests/io/parser/common/test_read_errors.py | 3 +-- pandas/tests/io/parser/test_quoting.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/tests/io/parser/common/test_read_errors.py b/pandas/tests/io/parser/common/test_read_errors.py index 57db65b42464b..47f1052808e0c 100644 --- a/pandas/tests/io/parser/common/test_read_errors.py +++ b/pandas/tests/io/parser/common/test_read_errors.py @@ -12,7 +12,6 @@ import numpy as np import pytest -from pandas.compat import PY311 from pandas.errors import ( EmptyDataError, ParserError, @@ -231,7 +230,7 @@ def test_null_byte_char(all_parsers): names = ["a", "b"] parser = all_parsers - if parser.engine == "c" or PY311: + if parser.engine == "c": expected = DataFrame([[np.nan, "foo"]], columns=names) out = parser.read_csv(StringIO(data), names=names) tm.assert_frame_equal(out, expected) diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index 8326a07ac5aee..456dd049d2f4a 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -8,7 +8,6 @@ import pytest -from pandas.compat import PY311 from pandas.errors import ParserError from pandas import DataFrame From 77dab8a683903d47e86587142058d4c6a4aea35a Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 6 Aug 2022 07:49:25 -0700 Subject: [PATCH 11/26] maybe fix errors --- pandas/tests/io/parser/common/test_read_errors.py | 3 ++- pandas/tests/io/parser/test_quoting.py | 10 ++++++++-- pandas/tests/scalar/test_nat.py | 2 -- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/parser/common/test_read_errors.py b/pandas/tests/io/parser/common/test_read_errors.py index 47f1052808e0c..48b8efa0e5adf 100644 --- a/pandas/tests/io/parser/common/test_read_errors.py +++ b/pandas/tests/io/parser/common/test_read_errors.py @@ -12,6 +12,7 @@ import numpy as np import pytest +from pandas.compat import PY311 from pandas.errors import ( EmptyDataError, ParserError, @@ -230,7 +231,7 @@ def test_null_byte_char(all_parsers): names = ["a", "b"] parser = all_parsers - if parser.engine == "c": + if parser.engine == "c" or (parser.engine == "python" and PY311): expected = DataFrame([[np.nan, "foo"]], columns=names) out = parser.read_csv(StringIO(data), names=names) tm.assert_frame_equal(out, expected) diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index a1aba949e74fe..112983c67fe04 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -8,6 +8,7 @@ import pytest +from pandas.compat import PY311 from pandas.errors import ParserError from pandas import DataFrame @@ -80,11 +81,16 @@ def test_null_quote_char(all_parsers, quoting, quote_char): if quoting != csv.QUOTE_NONE: # Sanity checking. - msg = "quotechar must be set if quoting enabled" + msg = ( + "quotechar must be set if quoting enabled" + if PY311 and all_parsers.engine == "python" + else "quotechar must be set if quoting enabled" + ) with pytest.raises(TypeError, match=msg): parser.read_csv(StringIO(data), **kwargs) - else: + elif not (PY311 and all_parsers.engine == "python"): + # Python 3.11+ doesn't support no quote chars in their csv parsers expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"]) result = parser.read_csv(StringIO(data), **kwargs) tm.assert_frame_equal(result, expected) diff --git a/pandas/tests/scalar/test_nat.py b/pandas/tests/scalar/test_nat.py index 1e7cdecfe442d..1a07c02f4024a 100644 --- a/pandas/tests/scalar/test_nat.py +++ b/pandas/tests/scalar/test_nat.py @@ -9,7 +9,6 @@ import pytz from pandas._libs.tslibs import iNaT -from pandas.compat import PY311 from pandas.core.dtypes.common import is_datetime64_any_dtype @@ -315,7 +314,6 @@ def test_overlap_public_nat_methods(klass, expected): assert _get_overlap_public_nat_methods(klass) == expected -@pytest.mark.xfail(PY311, reason="error in Python 3.11", raises=TypeError) @pytest.mark.parametrize( "compare", ( From bc1a776224d8dae8f3d135042d6eed7cd2a54322 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sat, 6 Aug 2022 10:32:55 -0700 Subject: [PATCH 12/26] update --- pandas/tests/io/parser/common/test_read_errors.py | 2 +- pandas/tests/io/parser/test_quoting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/parser/common/test_read_errors.py b/pandas/tests/io/parser/common/test_read_errors.py index 48b8efa0e5adf..93356840bb8db 100644 --- a/pandas/tests/io/parser/common/test_read_errors.py +++ b/pandas/tests/io/parser/common/test_read_errors.py @@ -234,7 +234,7 @@ def test_null_byte_char(all_parsers): if parser.engine == "c" or (parser.engine == "python" and PY311): expected = DataFrame([[np.nan, "foo"]], columns=names) out = parser.read_csv(StringIO(data), names=names) - tm.assert_frame_equal(out, expected) + tm.assert_frame_equal(out, expected, check_dtype=False) # debug else: msg = "NULL byte detected" with pytest.raises(ParserError, match=msg): diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index 112983c67fe04..0d199e2482603 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -82,7 +82,7 @@ def test_null_quote_char(all_parsers, quoting, quote_char): if quoting != csv.QUOTE_NONE: # Sanity checking. msg = ( - "quotechar must be set if quoting enabled" + '"quotechar" must be a 1-character string' if PY311 and all_parsers.engine == "python" else "quotechar must be set if quoting enabled" ) From ad5eef17d06fb07dde4917b33a9f410727cf1c62 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Sun, 7 Aug 2022 08:36:29 -0700 Subject: [PATCH 13/26] try something --- pandas/tests/io/parser/common/test_read_errors.py | 12 +++++++++--- pandas/tests/io/parser/test_quoting.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/parser/common/test_read_errors.py b/pandas/tests/io/parser/common/test_read_errors.py index 93356840bb8db..64feef6e27674 100644 --- a/pandas/tests/io/parser/common/test_read_errors.py +++ b/pandas/tests/io/parser/common/test_read_errors.py @@ -225,16 +225,22 @@ def test_read_csv_wrong_num_columns(all_parsers): parser.read_csv(StringIO(data)) -def test_null_byte_char(all_parsers): +def test_null_byte_char(request, all_parsers): # see gh-2741 data = "\x00,foo" names = ["a", "b"] parser = all_parsers - if parser.engine == "c" or (parser.engine == "python" and PY311): + if parser.engine in ["c", "python"]: + if parser.engine == "python" and PY311: + request.node.add_marker( + pytest.mark.xfail( + reason="In Python 3.11, this is read as an empty character not null" + ) + ) expected = DataFrame([[np.nan, "foo"]], columns=names) out = parser.read_csv(StringIO(data), names=names) - tm.assert_frame_equal(out, expected, check_dtype=False) # debug + tm.assert_frame_equal(out, expected) else: msg = "NULL byte detected" with pytest.raises(ParserError, match=msg): diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index 0d199e2482603..2d4bf338d2915 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -83,7 +83,7 @@ def test_null_quote_char(all_parsers, quoting, quote_char): # Sanity checking. msg = ( '"quotechar" must be a 1-character string' - if PY311 and all_parsers.engine == "python" + if PY311 and all_parsers.engine == "python" and quote_char == "" else "quotechar must be set if quoting enabled" ) From 71ab88da5ebdc14cda81ae3c1db6cf04b3b12b50 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 8 Aug 2022 06:34:01 -0700 Subject: [PATCH 14/26] pin hypothesis --- .github/workflows/python-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 1c9508c6ec440..5139e4758d7c5 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -55,7 +55,7 @@ jobs: python -m pip install git+https://github.com/cython/cython.git python -m pip install git+https://github.com/numpy/numpy.git python -m pip install git+https://github.com/nedbat/coveragepy.git - python -m pip install python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip install python-dateutil pytz hypothesis==6.52.3 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list - name: Build Pandas From aca30a08448515080d91d988227b8ef5f275e458 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 8 Aug 2022 14:10:40 -0700 Subject: [PATCH 15/26] Don't check parentheses equality --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 70adbd3c083af..0a8d89143369e 100755 --- a/setup.py +++ b/setup.py @@ -335,7 +335,7 @@ def run(self): else: # PANDAS_CI=1 is set in CI if os.environ.get("PANDAS_CI", "0") == "1": - extra_compile_args.append("-Werror") + extra_compile_args.append(["-Werror", "-Wno-parentheses-equality"]) if debugging_symbols_requested: extra_compile_args.append("-g") extra_compile_args.append("-UNDEBUG") From 11c9269e62a786d9bcc2588bbfba925352dd9b58 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 8 Aug 2022 14:19:27 -0700 Subject: [PATCH 16/26] typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0a8d89143369e..c1b087bc78e31 100755 --- a/setup.py +++ b/setup.py @@ -335,7 +335,7 @@ def run(self): else: # PANDAS_CI=1 is set in CI if os.environ.get("PANDAS_CI", "0") == "1": - extra_compile_args.append(["-Werror", "-Wno-parentheses-equality"]) + extra_compile_args += ["-Werror", "-Wno-parentheses-equality"] if debugging_symbols_requested: extra_compile_args.append("-g") extra_compile_args.append("-UNDEBUG") From 5cfb0719e06a383f2292dac524cf2db955b886d5 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 8 Aug 2022 14:39:56 -0700 Subject: [PATCH 17/26] pin cython --- .github/workflows/python-dev.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 5139e4758d7c5..b714e4d2988d1 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -52,10 +52,9 @@ jobs: run: | python --version python -m pip install --upgrade pip setuptools wheel - python -m pip install git+https://github.com/cython/cython.git python -m pip install git+https://github.com/numpy/numpy.git python -m pip install git+https://github.com/nedbat/coveragepy.git - python -m pip install python-dateutil pytz hypothesis==6.52.3 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip install python-dateutil pytz cython hypothesis==6.52.3 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list - name: Build Pandas From 668c53eda8c1705e91bf351686f54186c385cf34 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 8 Aug 2022 20:33:09 -0700 Subject: [PATCH 18/26] correctly pin hypothesis --- .github/workflows/python-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index b714e4d2988d1..fd7341896c565 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -54,7 +54,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install git+https://github.com/numpy/numpy.git python -m pip install git+https://github.com/nedbat/coveragepy.git - python -m pip install python-dateutil pytz cython hypothesis==6.52.3 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list - name: Build Pandas From 827a6483d875a88d0598c2c42eced43a102f0f8d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 10 Aug 2022 14:48:24 -0700 Subject: [PATCH 19/26] adjust test --- pandas/tests/io/parser/common/test_read_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/parser/common/test_read_errors.py b/pandas/tests/io/parser/common/test_read_errors.py index 64feef6e27674..f52af109626e9 100644 --- a/pandas/tests/io/parser/common/test_read_errors.py +++ b/pandas/tests/io/parser/common/test_read_errors.py @@ -231,7 +231,7 @@ def test_null_byte_char(request, all_parsers): names = ["a", "b"] parser = all_parsers - if parser.engine in ["c", "python"]: + if parser.engine == "c" or (parser.engine == "python" and PY311): if parser.engine == "python" and PY311: request.node.add_marker( pytest.mark.xfail( From 5fb3daabd808a5bba41b856de7a72b2d6d3a4fc2 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:41:03 -0700 Subject: [PATCH 20/26] remove compiler flag --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c1b087bc78e31..70adbd3c083af 100755 --- a/setup.py +++ b/setup.py @@ -335,7 +335,7 @@ def run(self): else: # PANDAS_CI=1 is set in CI if os.environ.get("PANDAS_CI", "0") == "1": - extra_compile_args += ["-Werror", "-Wno-parentheses-equality"] + extra_compile_args.append("-Werror") if debugging_symbols_requested: extra_compile_args.append("-g") extra_compile_args.append("-UNDEBUG") From b1a58fe6d98529741e9ece0fbe780cb7bf658920 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Thu, 11 Aug 2022 10:11:27 -0700 Subject: [PATCH 21/26] typo'ed --- pandas/tests/io/parser/test_quoting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/parser/test_quoting.py b/pandas/tests/io/parser/test_quoting.py index 2d4bf338d2915..025a612dc47d2 100644 --- a/pandas/tests/io/parser/test_quoting.py +++ b/pandas/tests/io/parser/test_quoting.py @@ -90,7 +90,7 @@ def test_null_quote_char(all_parsers, quoting, quote_char): with pytest.raises(TypeError, match=msg): parser.read_csv(StringIO(data), **kwargs) elif not (PY311 and all_parsers.engine == "python"): - # Python 3.11+ doesn't support no quote chars in their csv parsers + # Python 3.11+ doesn't support null/blank quote chars in their csv parsers expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"]) result = parser.read_csv(StringIO(data), **kwargs) tm.assert_frame_equal(result, expected) From 354b12c8102872762b564424de47f47e08d79446 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 15 Aug 2022 10:00:13 -0700 Subject: [PATCH 22/26] fix tests --- pandas/tests/indexes/datetimes/test_constructors.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index 086cb18dbe463..26b98d821b6ba 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -1137,8 +1137,10 @@ def test_timestamp_constructor_retain_fold(tz, fold): _tzs = ["dateutil/Europe/London"] if PY39: - _tzs = ["dateutil/Europe/London", zoneinfo.ZoneInfo("Europe/London")] - + try: + _tzs = ["dateutil/Europe/London", zoneinfo.ZoneInfo("Europe/London")] + except zoneinfo.ZoneInfoNotFoundError: + pass @pytest.mark.parametrize("tz", _tzs) @pytest.mark.parametrize( From 54023de8e62aa17f477e95b306564b2b3c47ee0f Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Mon, 15 Aug 2022 11:24:45 -0700 Subject: [PATCH 23/26] linter --- pandas/tests/indexes/datetimes/test_constructors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/indexes/datetimes/test_constructors.py b/pandas/tests/indexes/datetimes/test_constructors.py index 26b98d821b6ba..1d161630b1356 100644 --- a/pandas/tests/indexes/datetimes/test_constructors.py +++ b/pandas/tests/indexes/datetimes/test_constructors.py @@ -1142,6 +1142,7 @@ def test_timestamp_constructor_retain_fold(tz, fold): except zoneinfo.ZoneInfoNotFoundError: pass + @pytest.mark.parametrize("tz", _tzs) @pytest.mark.parametrize( "ts_input,fold_out", From 9adcc80e66653b77e4de2b128cefd47d6f7b07d0 Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 17 Aug 2022 11:05:35 -0700 Subject: [PATCH 24/26] clarify freeze/unfreeze instructions --- .github/workflows/python-dev.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index fd7341896c565..27e66aeb996b5 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -1,3 +1,23 @@ +# This workflow may or may not run depending on the state of the next +# unreleased Python version. DO NOT DELETE IT. +# +# In general, this file will remain frozen(present, but not running) until: +# - The next unreleased Python version has released beta 1 +# - This version should be available on Github Actions. +# - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil) +# support that unreleased Python version. +# To unfreeze, comment out the ``if: false`` condition, and make sure you update +# the name of the workflow and Python version in actions/setup-python to: '3.12-dev' +# +# After it has been unfrozen, this file should remain unfrozen(present, and running) until: +# - The next Python version has been officially released. +# OR +# - Most/All of our optional dependencies support Python 3.11 AND +# - The next Python version has released a rc(we are guaranteed a stable ABI). +# To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs +# to the corresponding posix/windows-macos/sdist etc. workflows. +# Feel free to modify this comment as necessary. + name: Python Dev on: @@ -24,6 +44,7 @@ permissions: jobs: build: + # if: false # Uncomment this to freeze the workflow, comment it to unfreeze runs-on: ${{ matrix.os }} strategy: fail-fast: false From ce6a0a4ad9fb4f73872e338673a6e21650d6293a Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 17 Aug 2022 11:42:36 -0700 Subject: [PATCH 25/26] whitespace --- .github/workflows/python-dev.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 27e66aeb996b5..f495f46dfdc63 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -1,17 +1,17 @@ # This workflow may or may not run depending on the state of the next # unreleased Python version. DO NOT DELETE IT. -# +# # In general, this file will remain frozen(present, but not running) until: # - The next unreleased Python version has released beta 1 # - This version should be available on Github Actions. -# - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil) +# - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil) # support that unreleased Python version. -# To unfreeze, comment out the ``if: false`` condition, and make sure you update +# To unfreeze, comment out the ``if: false`` condition, and make sure you update # the name of the workflow and Python version in actions/setup-python to: '3.12-dev' # # After it has been unfrozen, this file should remain unfrozen(present, and running) until: # - The next Python version has been officially released. -# OR +# OR # - Most/All of our optional dependencies support Python 3.11 AND # - The next Python version has released a rc(we are guaranteed a stable ABI). # To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs From 0b3278227e75439c4fe86dee826c90d2942b8c7d Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 17 Aug 2022 12:18:54 -0700 Subject: [PATCH 26/26] one more --- .github/workflows/python-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index f495f46dfdc63..580cafd6e4949 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -8,7 +8,7 @@ # support that unreleased Python version. # To unfreeze, comment out the ``if: false`` condition, and make sure you update # the name of the workflow and Python version in actions/setup-python to: '3.12-dev' -# +# # After it has been unfrozen, this file should remain unfrozen(present, and running) until: # - The next Python version has been officially released. # OR