From 72cbb698476122648e702fc576a2af9c3041f2c8 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Thu, 17 Nov 2022 15:20:09 -0500 Subject: [PATCH 1/9] WIP: Python 3.11 support --- .github/workflows/test.yml | 2 +- pandas-stubs/core/arrays/arrow/dtype.pyi | 11 ++++++++++- pyproject.toml | 11 ++++++----- tests/test_pandas.py | 16 +++++++++------- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a846ab50..7f271f8c5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ['3.8', '3.9', '3.10.6'] + python-version: ['3.8', '3.9', '3.10.6', '3.11'] steps: - uses: actions/checkout@v3 diff --git a/pandas-stubs/core/arrays/arrow/dtype.pyi b/pandas-stubs/core/arrays/arrow/dtype.pyi index 041012a7a..5934aeb25 100644 --- a/pandas-stubs/core/arrays/arrow/dtype.pyi +++ b/pandas-stubs/core/arrays/arrow/dtype.pyi @@ -1,4 +1,13 @@ -import pyarrow as pa +import sys +from typing import ( + Any, + TypeAlias, +) + +if sys.version_info < (3, 11): + import pyarrow as pa +else: + pa: TypeAlias = Any from pandas.core.dtypes.base import StorageExtensionDtype diff --git a/pyproject.toml b/pyproject.toml index e1fc29648..92a34ff22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering" ] packages = [ @@ -30,12 +31,12 @@ packages = [ "Documentation" = "https://pandas.pydata.org/pandas-docs/stable" [tool.poetry.dependencies] -python = ">=3.8,<3.11" +python = ">=3.8,<3.12" types-pytz = ">= 2022.1.1" [tool.poetry.dev-dependencies] mypy = "0.990" -pyarrow = ">=9.0.0" +pyarrow = { version = ">=9.0.0", python = "<3.11" } pytest = ">=7.1.2" pyright = ">=1.1.278" poethepoet = "0.16.0" @@ -47,9 +48,9 @@ pre-commit = ">=2.19.0" black = ">=22.8.0" isort = ">=5.10.1" openpyxl = ">=3.0.10" -tables = ">=3.7.0" -lxml = ">=4.7.1,<4.9.0" -pyreadstat = ">=1.1.9" +tables = { version = ">=3.7.0", python = "<3.11" } +lxml = { version = ">=4.7.1,<4.9.0", python = "<3.11" } +pyreadstat = { version = ">=1.1.9", python = "<3.11" } xlrd = ">=2.0.1" pyxlsb = ">=1.0.9" odfpy = ">=1.4.1" diff --git a/tests/test_pandas.py b/tests/test_pandas.py index 456f94947..e0986aef5 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -2,6 +2,7 @@ import datetime as dt import random +import sys from typing import ( TYPE_CHECKING, Any, @@ -464,14 +465,15 @@ def test_crosstab() -> None: def test_arrow_dtype() -> None: pytest.importorskip("pyarrow") - import pyarrow as pa + if sys.version_info < (3, 11): + import pyarrow as pa - check( - assert_type( - pd.ArrowDtype(pa.timestamp("s", tz="America/New_York")), pd.ArrowDtype - ), - pd.ArrowDtype, - ) + check( + assert_type( + pd.ArrowDtype(pa.timestamp("s", tz="America/New_York")), pd.ArrowDtype + ), + pd.ArrowDtype, + ) def test_hashing(): From 874d9013f1033e242fd707c378fa354c722ed98e Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sat, 19 Nov 2022 23:41:16 -0500 Subject: [PATCH 2/9] put in temp warning in test_frame.py --- pandas-stubs/core/arrays/arrow/dtype.pyi | 7 ++--- pyproject.toml | 2 +- tests/__init__.py | 35 ++++++++++++++++++++++-- tests/test_frame.py | 11 +++++++- tests/test_io.py | 22 +++++++++++++++ 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/pandas-stubs/core/arrays/arrow/dtype.pyi b/pandas-stubs/core/arrays/arrow/dtype.pyi index 5934aeb25..751067176 100644 --- a/pandas-stubs/core/arrays/arrow/dtype.pyi +++ b/pandas-stubs/core/arrays/arrow/dtype.pyi @@ -1,8 +1,7 @@ import sys -from typing import ( - Any, - TypeAlias, -) +from typing import Any + +from typing_extensions import TypeAlias if sys.version_info < (3, 11): import pyarrow as pa diff --git a/pyproject.toml b/pyproject.toml index 92a34ff22..34626fe02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ isort = ">=5.10.1" openpyxl = ">=3.0.10" tables = { version = ">=3.7.0", python = "<3.11" } lxml = { version = ">=4.7.1,<4.9.0", python = "<3.11" } -pyreadstat = { version = ">=1.1.9", python = "<3.11" } +pyreadstat = ">=1.2.0" xlrd = ">=2.0.1" pyxlsb = ">=1.0.9" odfpy = ">=1.4.1" diff --git a/tests/__init__.py b/tests/__init__.py index 43008807a..0bd199f33 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,6 +6,7 @@ ) import os import platform +import sys from typing import ( TYPE_CHECKING, Final, @@ -21,6 +22,21 @@ WINDOWS = os.name == "nt" or "cygwin" in platform.system().lower() PD_LTE_15 = Version(pd.__version__) < Version("1.5.999") +arrow_skip = pytest.mark.skipif( + sys.version_info >= (3, 11), reason="pyarrow is not available for 3.11 yet" +) +"""This is only needed temporarily due to no wheels being available for arrow on 3.11""" + +lxml_skip = pytest.mark.skipif( + sys.version_info >= (3, 11), reason="lxml is not available for 3.11 yet" +) +"""This is only needed temporarily due to no wheels being available for lxml on 3.11""" + +pytables_skip = pytest.mark.skipif( + sys.version_info >= (3, 11), reason="pytables is not available for 3.11 yet" +) +"""This is only needed temporarily due to no wheels being available for pytables on 3.11""" + def check(actual: T, klass: type, dtype: type | None = None, attr: str = "left") -> T: if not isinstance(actual, klass): @@ -44,6 +60,7 @@ def pytest_warns_bounded( match: str, lower: str | None = None, upper: str | None = None, + version_str: str | None = None, ) -> AbstractContextManager: """ Version conditional pytest.warns context manager @@ -62,11 +79,14 @@ def pytest_warns_bounded( The lower bound of the version to check for the warning. upper : str, optional The upper bound of the version to check for the warning. + version_str: str, optional + The version string to use. If None, then uses the pandas version. + Can be used to check a python version as well Notes ----- The lower and upper bounds are exclusive so that a pytest.warns context - manager is returned if lower < pd.__version__ < upper. + manager is returned if lower < version_str < upper. Examples -------- @@ -85,10 +105,21 @@ def pytest_warns_bounded( ): # Versions between 1.3.x and 1.5.x will raise an error pass + + with pytest_warns_bounded( + UserWarning, match="foo", lower="3.10", + version_str = platform.python_version() + ): + # Python version 3.11 and above will raise an error + # if the warning is not issued + pass """ lb = Version("0.0.0") if lower is None else Version(lower) ub = Version("9999.0.0") if upper is None else Version(upper) - current = Version(pd.__version__) + if version_str is None: + current = Version(pd.__version__) + else: + current = Version(version_str) if lb < current < ub: return pytest.warns(warning, match=match) else: diff --git a/tests/test_frame.py b/tests/test_frame.py index b7108d1a5..ea8983607 100644 --- a/tests/test_frame.py +++ b/tests/test_frame.py @@ -6,6 +6,7 @@ import io import itertools from pathlib import Path +import platform from typing import ( TYPE_CHECKING, Any, @@ -1757,7 +1758,15 @@ class MyDataFrame(pd.DataFrame, Generic[T]): def func() -> MyDataFrame[int]: return MyDataFrame[int]({"foo": [1, 2, 3]}) - func() + # This should be fixed in pandas 1.5.2, if + # https://github.com/pandas-dev/pandas/pull/49736 is included + with pytest_warns_bounded( + UserWarning, + "Pandas doesn't allow columns to be created", + lower="3.10", + version_str=platform.python_version(), + ): + func() def test_to_xarray(): diff --git a/tests/test_io.py b/tests/test_io.py index 3a8bee13d..276940a1a 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -65,10 +65,17 @@ from pandas.io.sas.sas_xport import XportReader from pandas.io.stata import StataReader +from . import ( + arrow_skip, + lxml_skip, + pytables_skip, +) + DF = DataFrame({"a": [1, 2, 3], "b": [0.0, 0.0, 0.0]}) CWD = os.path.split(os.path.abspath(__file__))[0] +@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc(): with ensure_clean() as path: @@ -76,6 +83,7 @@ def test_orc(): check(assert_type(read_orc(path), DataFrame), DataFrame) +@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_path(): with ensure_clean() as path: @@ -84,6 +92,7 @@ def test_orc_path(): check(assert_type(read_orc(pathlib_path), DataFrame), DataFrame) +@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_buffer(): with ensure_clean() as path: @@ -96,6 +105,7 @@ def test_orc_buffer(): file_r.close() +@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_columns(): with ensure_clean() as path: @@ -103,11 +113,13 @@ def test_orc_columns(): check(assert_type(read_orc(path, columns=["a"]), DataFrame), DataFrame) +@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_bytes(): check(assert_type(DF.to_orc(index=False), bytes), bytes) +@lxml_skip def test_xml(): with ensure_clean() as path: check(assert_type(DF.to_xml(path), None), type(None)) @@ -116,6 +128,7 @@ def test_xml(): check(assert_type(read_xml(f), DataFrame), DataFrame) +@lxml_skip def test_xml_str(): with ensure_clean() as path: check(assert_type(DF.to_xml(), str), str) @@ -282,12 +295,14 @@ def test_sas_xport() -> None: pass +@pytables_skip def test_hdf(): with ensure_clean() as path: check(assert_type(DF.to_hdf(path, "df"), None), type(None)) check(assert_type(read_hdf(path), Union[DataFrame, Series]), DataFrame) +@pytables_skip def test_hdfstore(): with ensure_clean() as path: store = HDFStore(path, model="w") @@ -329,6 +344,7 @@ def test_hdfstore(): store.close() +@pytables_skip def test_read_hdf_iterator(): with ensure_clean() as path: check(assert_type(DF.to_hdf(path, "df", format="table"), None), type(None)) @@ -343,6 +359,7 @@ def test_read_hdf_iterator(): ti.close() +@pytables_skip def test_hdf_context_manager(): with ensure_clean() as path: check(assert_type(DF.to_hdf(path, "df", format="table"), None), type(None)) @@ -351,6 +368,7 @@ def test_hdf_context_manager(): check(assert_type(store.get("df"), Union[DataFrame, Series]), DataFrame) +@pytables_skip def test_hdf_series(): s = DF["a"] with ensure_clean() as path: @@ -392,6 +410,7 @@ def test_json_chunk(): check(assert_type(DF.to_json(), str), str) +@arrow_skip def test_parquet(): with ensure_clean() as path: check(assert_type(DF.to_parquet(path), None), type(None)) @@ -399,6 +418,7 @@ def test_parquet(): check(assert_type(DF.to_parquet(), bytes), bytes) +@arrow_skip def test_parquet_options(): with ensure_clean(".parquet") as path: check( @@ -408,6 +428,7 @@ def test_parquet_options(): check(assert_type(read_parquet(path), DataFrame), DataFrame) +@arrow_skip def test_feather(): with ensure_clean() as path: check(assert_type(DF.to_feather(path), None), type(None)) @@ -795,6 +816,7 @@ def test_read_sql_query_generator(): con.close() +@lxml_skip def test_read_html(): check(assert_type(DF.to_html(), str), str) with ensure_clean() as path: From 3bd3daab78078ac4ee16f441fd9eb31d7aa25e1e Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sun, 20 Nov 2022 00:00:22 -0500 Subject: [PATCH 3/9] fix pyright up to 1.1.279 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34626fe02..3afc9aba0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ types-pytz = ">= 2022.1.1" mypy = "0.990" pyarrow = { version = ">=9.0.0", python = "<3.11" } pytest = ">=7.1.2" -pyright = ">=1.1.278" +pyright = "<=1.1.279" poethepoet = "0.16.0" loguru = ">=0.6.0" pandas = "1.5.1" From acc024971997acaa709875d3d52f90d27e22cfdd Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sun, 20 Nov 2022 00:07:40 -0500 Subject: [PATCH 4/9] fix bounds test in test_frame.py:test_generic --- tests/test_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_frame.py b/tests/test_frame.py index ea8983607..250959fd8 100644 --- a/tests/test_frame.py +++ b/tests/test_frame.py @@ -1763,7 +1763,7 @@ def func() -> MyDataFrame[int]: with pytest_warns_bounded( UserWarning, "Pandas doesn't allow columns to be created", - lower="3.10", + lower="3.11", version_str=platform.python_version(), ): func() From a6f8fec53f66b2af628a2b07d748092371ce58b9 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sun, 20 Nov 2022 00:22:03 -0500 Subject: [PATCH 5/9] change bound on test to 3.10.99 --- tests/test_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_frame.py b/tests/test_frame.py index 250959fd8..6f41544c1 100644 --- a/tests/test_frame.py +++ b/tests/test_frame.py @@ -1763,7 +1763,7 @@ def func() -> MyDataFrame[int]: with pytest_warns_bounded( UserWarning, "Pandas doesn't allow columns to be created", - lower="3.11", + lower="3.10.99", version_str=platform.python_version(), ): func() From 5eada8578eda57afae7088358208770ef490adbd Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Sun, 20 Nov 2022 12:23:30 -0500 Subject: [PATCH 6/9] change strings to comments related to unavailability --- tests/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 0bd199f33..cf95bae10 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -25,17 +25,17 @@ arrow_skip = pytest.mark.skipif( sys.version_info >= (3, 11), reason="pyarrow is not available for 3.11 yet" ) -"""This is only needed temporarily due to no wheels being available for arrow on 3.11""" +# This is only needed temporarily due to no wheels being available for arrow on 3.11 lxml_skip = pytest.mark.skipif( sys.version_info >= (3, 11), reason="lxml is not available for 3.11 yet" ) -"""This is only needed temporarily due to no wheels being available for lxml on 3.11""" +# This is only needed temporarily due to no wheels being available for lxml on 3.11 pytables_skip = pytest.mark.skipif( sys.version_info >= (3, 11), reason="pytables is not available for 3.11 yet" ) -"""This is only needed temporarily due to no wheels being available for pytables on 3.11""" +# This is only needed temporarily due to no wheels being available for pytables on 3.11 def check(actual: T, klass: type, dtype: type | None = None, attr: str = "left") -> T: From 4fb94ccfc57f071208b335a420ed0cd3da88a48f Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Wed, 23 Nov 2022 21:50:39 -0500 Subject: [PATCH 7/9] test with pandas 1.5.2, pyright 1.1.281 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3afc9aba0..bcda6508d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,10 @@ types-pytz = ">= 2022.1.1" mypy = "0.990" pyarrow = { version = ">=9.0.0", python = "<3.11" } pytest = ">=7.1.2" -pyright = "<=1.1.279" +pyright = ">=1.1.281" poethepoet = "0.16.0" loguru = ">=0.6.0" -pandas = "1.5.1" +pandas = "1.5.2" typing-extensions = ">=4.2.0" matplotlib = ">=3.5.1" pre-commit = ">=2.19.0" From 370f27c5e7ec3f4e5ff7161df23defe690b8565e Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Wed, 23 Nov 2022 22:30:12 -0500 Subject: [PATCH 8/9] remove pyarrow checks for python 3.11 --- pandas-stubs/core/arrays/arrow/dtype.pyi | 10 +--------- pyproject.toml | 2 +- tests/__init__.py | 5 ----- tests/test_io.py | 9 --------- tests/test_pandas.py | 16 +++++++--------- 5 files changed, 9 insertions(+), 33 deletions(-) diff --git a/pandas-stubs/core/arrays/arrow/dtype.pyi b/pandas-stubs/core/arrays/arrow/dtype.pyi index 751067176..041012a7a 100644 --- a/pandas-stubs/core/arrays/arrow/dtype.pyi +++ b/pandas-stubs/core/arrays/arrow/dtype.pyi @@ -1,12 +1,4 @@ -import sys -from typing import Any - -from typing_extensions import TypeAlias - -if sys.version_info < (3, 11): - import pyarrow as pa -else: - pa: TypeAlias = Any +import pyarrow as pa from pandas.core.dtypes.base import StorageExtensionDtype diff --git a/pyproject.toml b/pyproject.toml index bcda6508d..ed31d2421 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ types-pytz = ">= 2022.1.1" [tool.poetry.dev-dependencies] mypy = "0.990" -pyarrow = { version = ">=9.0.0", python = "<3.11" } +pyarrow = ">=10.0.1" pytest = ">=7.1.2" pyright = ">=1.1.281" poethepoet = "0.16.0" diff --git a/tests/__init__.py b/tests/__init__.py index cf95bae10..f41304829 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -22,11 +22,6 @@ WINDOWS = os.name == "nt" or "cygwin" in platform.system().lower() PD_LTE_15 = Version(pd.__version__) < Version("1.5.999") -arrow_skip = pytest.mark.skipif( - sys.version_info >= (3, 11), reason="pyarrow is not available for 3.11 yet" -) -# This is only needed temporarily due to no wheels being available for arrow on 3.11 - lxml_skip = pytest.mark.skipif( sys.version_info >= (3, 11), reason="lxml is not available for 3.11 yet" ) diff --git a/tests/test_io.py b/tests/test_io.py index 276940a1a..38f0bcffe 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -66,7 +66,6 @@ from pandas.io.stata import StataReader from . import ( - arrow_skip, lxml_skip, pytables_skip, ) @@ -75,7 +74,6 @@ CWD = os.path.split(os.path.abspath(__file__))[0] -@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc(): with ensure_clean() as path: @@ -83,7 +81,6 @@ def test_orc(): check(assert_type(read_orc(path), DataFrame), DataFrame) -@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_path(): with ensure_clean() as path: @@ -92,7 +89,6 @@ def test_orc_path(): check(assert_type(read_orc(pathlib_path), DataFrame), DataFrame) -@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_buffer(): with ensure_clean() as path: @@ -105,7 +101,6 @@ def test_orc_buffer(): file_r.close() -@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_columns(): with ensure_clean() as path: @@ -113,7 +108,6 @@ def test_orc_columns(): check(assert_type(read_orc(path, columns=["a"]), DataFrame), DataFrame) -@arrow_skip @pytest.mark.skipif(WINDOWS, reason="ORC not available on windows") def test_orc_bytes(): check(assert_type(DF.to_orc(index=False), bytes), bytes) @@ -410,7 +404,6 @@ def test_json_chunk(): check(assert_type(DF.to_json(), str), str) -@arrow_skip def test_parquet(): with ensure_clean() as path: check(assert_type(DF.to_parquet(path), None), type(None)) @@ -418,7 +411,6 @@ def test_parquet(): check(assert_type(DF.to_parquet(), bytes), bytes) -@arrow_skip def test_parquet_options(): with ensure_clean(".parquet") as path: check( @@ -428,7 +420,6 @@ def test_parquet_options(): check(assert_type(read_parquet(path), DataFrame), DataFrame) -@arrow_skip def test_feather(): with ensure_clean() as path: check(assert_type(DF.to_feather(path), None), type(None)) diff --git a/tests/test_pandas.py b/tests/test_pandas.py index e0986aef5..456f94947 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -2,7 +2,6 @@ import datetime as dt import random -import sys from typing import ( TYPE_CHECKING, Any, @@ -465,15 +464,14 @@ def test_crosstab() -> None: def test_arrow_dtype() -> None: pytest.importorskip("pyarrow") - if sys.version_info < (3, 11): - import pyarrow as pa + import pyarrow as pa - check( - assert_type( - pd.ArrowDtype(pa.timestamp("s", tz="America/New_York")), pd.ArrowDtype - ), - pd.ArrowDtype, - ) + check( + assert_type( + pd.ArrowDtype(pa.timestamp("s", tz="America/New_York")), pd.ArrowDtype + ), + pd.ArrowDtype, + ) def test_hashing(): From 8bdbd3e4621f3ffd1a71ea811b28a8cfff6b0109 Mon Sep 17 00:00:00 2001 From: Irv Lustig Date: Wed, 23 Nov 2022 22:48:18 -0500 Subject: [PATCH 9/9] update github action to get rid of github set-output warning --- .github/setup/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/setup/action.yaml b/.github/setup/action.yaml index fe6b568c1..4d27df037 100644 --- a/.github/setup/action.yaml +++ b/.github/setup/action.yaml @@ -21,7 +21,7 @@ runs: - name: Determine poetry version shell: bash - run: echo "::set-output name=VERSION::$(poetry --version)" + run: echo "{VERSION}=$(poetry --version)" id: poetry_version - name: Cache poetry.lock