diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index cedc2b85c2794..8afaf9154aacf 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -87,7 +87,7 @@ jobs: uses: ./.github/actions/build_pandas - name: Test - run: pytest -m "${{ env.PATTERN }}" -n 2 --dist=loadfile --cov=pandas --cov-report=xml pandas/tests/io + run: pytest -m "${{ env.PATTERN }}" -n 2 --dist=loadfile --cov=pandas --cov-report=xml pandas/tests/io --timeout=300 if: always() - name: Build Version diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 596c3b6df9d49..0f3aa5ff69db1 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -45,7 +45,7 @@ jobs: pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy pip install git+https://github.com/pytest-dev/pytest.git pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov + pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov pytest-timeout pip list - name: Build Pandas diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index eaac17d50c315..96859f0b9954e 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -12,12 +12,12 @@ jobs: py38_macos_1: ENV_FILE: ci/deps/azure-macos-38.yaml CONDA_PY: "38" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not single" PYTEST_TARGET: "pandas/tests/[a-h]*" py38_macos_2: ENV_FILE: ci/deps/azure-macos-38.yaml CONDA_PY: "38" - PATTERN: "not slow and not network" + PATTERN: "not slow and not network and not single" PYTEST_TARGET: "pandas/tests/[i-z]*" steps: diff --git a/ci/deps/actions-38-db-min.yaml b/ci/deps/actions-38-db-min.yaml index c93f791b7dba7..63d1e60f2c2b4 100644 --- a/ci/deps/actions-38-db-min.yaml +++ b/ci/deps/actions-38-db-min.yaml @@ -45,3 +45,5 @@ dependencies: - psycopg2=2.8.4 - pymysql=0.10.1 - sqlalchemy=1.3.11 + - pip: + - pytest-timeout diff --git a/ci/deps/actions-38-db.yaml b/ci/deps/actions-38-db.yaml index b4495fa6887f4..e9807981b1704 100644 --- a/ci/deps/actions-38-db.yaml +++ b/ci/deps/actions-38-db.yaml @@ -51,3 +51,4 @@ dependencies: - coverage - pandas-datareader - pyxlsb + - pytest-timeout diff --git a/ci/deps/actions-38-locale.yaml b/ci/deps/actions-38-locale.yaml index 46ebaadc3049a..01265876d3ae5 100644 --- a/ci/deps/actions-38-locale.yaml +++ b/ci/deps/actions-38-locale.yaml @@ -39,3 +39,4 @@ dependencies: - pip - pip: - pyxlsb + - pytest-timeout diff --git a/ci/deps/actions-38-locale_slow.yaml b/ci/deps/actions-38-locale_slow.yaml index e7276027f2a41..95f80aa994a70 100644 --- a/ci/deps/actions-38-locale_slow.yaml +++ b/ci/deps/actions-38-locale_slow.yaml @@ -28,3 +28,5 @@ dependencies: - xlsxwriter=1.2.2 - xlwt=1.3.0 - html5lib=1.1 + - pip: + - pytest-timeout diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index d666bc3b555f5..91c3cbe5c8326 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -29,3 +29,5 @@ dependencies: - xlsxwriter=1.2.2 - xlwt=1.3.0 - html5lib=1.1 + - pip: + - pytest-timeout diff --git a/ci/deps/actions-38-slow.yaml b/ci/deps/actions-38-slow.yaml index 08900a31fe27c..372e5e39588d6 100644 --- a/ci/deps/actions-38-slow.yaml +++ b/ci/deps/actions-38-slow.yaml @@ -36,3 +36,5 @@ dependencies: - moto - flask - numba + - pip: + - pytest-timeout diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 86b038ff7d4b6..0e0f8778e46a8 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -18,3 +18,5 @@ dependencies: - nomkl - pytz - tabulate==0.8.7 + - pip: + - pytest-timeout diff --git a/ci/deps/actions-39-numpydev.yaml b/ci/deps/actions-39-numpydev.yaml index 03181a9d71d1d..357b927578959 100644 --- a/ci/deps/actions-39-numpydev.yaml +++ b/ci/deps/actions-39-numpydev.yaml @@ -20,3 +20,4 @@ dependencies: - "--pre" - "numpy" - "scipy" + - pytest-timeout diff --git a/ci/deps/actions-39-slow.yaml b/ci/deps/actions-39-slow.yaml index dbfcf98a3203b..22566003b7966 100644 --- a/ci/deps/actions-39-slow.yaml +++ b/ci/deps/actions-39-slow.yaml @@ -41,3 +41,4 @@ dependencies: - pip - pip: - pyxlsb + - pytest-timeout diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index a78fb30019b53..d1544e35ea03b 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -40,3 +40,4 @@ dependencies: - pip - pip: - pyxlsb + - pytest-timeout diff --git a/ci/deps/azure-macos-38.yaml b/ci/deps/azure-macos-38.yaml index 029d088362aa9..00ba4c777089b 100644 --- a/ci/deps/azure-macos-38.yaml +++ b/ci/deps/azure-macos-38.yaml @@ -35,3 +35,4 @@ dependencies: - cython>=0.29.21 - pyreadstat - pyxlsb + - pytest-timeout diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml index c56496bce7d6c..b213cf2c1ab10 100644 --- a/ci/deps/azure-windows-38.yaml +++ b/ci/deps/azure-windows-38.yaml @@ -34,3 +34,5 @@ dependencies: - xlrd - xlsxwriter - xlwt + - pip: + - pytest-timeout diff --git a/ci/deps/azure-windows-39.yaml b/ci/deps/azure-windows-39.yaml index c4d376fd2a909..dafc7a752d086 100644 --- a/ci/deps/azure-windows-39.yaml +++ b/ci/deps/azure-windows-39.yaml @@ -41,3 +41,4 @@ dependencies: - pip - pip: - pyxlsb + - pytest-timeout diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 53a2bf151d3bf..46f674583c925 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET --timeout=60" if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then # GH#37455 windows py38 build appears to be running out of memory diff --git a/environment.yml b/environment.yml index 733bd06fbe12f..3822139d4a4c9 100644 --- a/environment.yml +++ b/environment.yml @@ -63,6 +63,7 @@ dependencies: - pytest-xdist>=1.31 - pytest-asyncio - pytest-instafail + - pytest-timeout # downstream tests - seaborn diff --git a/pandas/tests/io/parser/common/test_index.py b/pandas/tests/io/parser/common/test_index.py index a37bd010d0e1b..bed9c4158754b 100644 --- a/pandas/tests/io/parser/common/test_index.py +++ b/pandas/tests/io/parser/common/test_index.py @@ -16,6 +16,7 @@ import pandas._testing as tm xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail") +skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip") @pytest.mark.parametrize( @@ -271,7 +272,7 @@ def test_empty_with_index(all_parsers): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow def test_empty_with_multi_index(all_parsers): # see gh-10467 data = "x,y,z" @@ -284,7 +285,7 @@ def test_empty_with_multi_index(all_parsers): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow def test_empty_with_reversed_multi_index(all_parsers): data = "x,y,z" parser = all_parsers diff --git a/pandas/tests/io/parser/common/test_ints.py b/pandas/tests/io/parser/common/test_ints.py index 0026bdc3c0ae3..75aae0e31f6b6 100644 --- a/pandas/tests/io/parser/common/test_ints.py +++ b/pandas/tests/io/parser/common/test_ints.py @@ -13,7 +13,6 @@ ) import pandas._testing as tm -xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail") skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip") @@ -119,7 +118,7 @@ def test_int64_min_issues(all_parsers): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize("conv", [None, np.int64, np.uint64]) def test_int64_overflow(all_parsers, conv): data = """ID @@ -163,7 +162,7 @@ def test_int64_overflow(all_parsers, conv): parser.read_csv(StringIO(data), converters={"ID": conv}) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize( "val", [np.iinfo(np.uint64).max, np.iinfo(np.int64).max, np.iinfo(np.int64).min] ) @@ -177,7 +176,7 @@ def test_int64_uint64_range(all_parsers, val): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize( "val", [np.iinfo(np.uint64).max + 1, np.iinfo(np.int64).min - 1] ) @@ -191,7 +190,7 @@ def test_outside_int64_uint64_range(all_parsers, val): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize("exp_data", [[str(-1), str(2 ** 63)], [str(2 ** 63), str(-1)]]) def test_numeric_range_too_wide(all_parsers, exp_data): # No numerical dtype can hold both negative and uint64 diff --git a/pandas/tests/io/parser/conftest.py b/pandas/tests/io/parser/conftest.py index efef06ef28574..2070057aff10b 100644 --- a/pandas/tests/io/parser/conftest.py +++ b/pandas/tests/io/parser/conftest.py @@ -82,10 +82,10 @@ def csv1(datapath): return os.path.join(datapath("io", "data", "csv"), "test1.csv") -_cParserHighMemory = CParserHighMemory() -_cParserLowMemory = CParserLowMemory() -_pythonParser = PythonParser() -_pyarrowParser = PyArrowParser() +_cParserHighMemory = CParserHighMemory +_cParserLowMemory = CParserLowMemory +_pythonParser = PythonParser +_pyarrowParser = PyArrowParser _py_parsers_only = [_pythonParser] _c_parsers_only = [_cParserHighMemory, _cParserLowMemory] @@ -105,13 +105,14 @@ def all_parsers(request): """ Fixture all of the CSV parsers. """ - if request.param.engine == "pyarrow": + parser = request.param() + if parser.engine == "pyarrow": pytest.importorskip("pyarrow", VERSIONS["pyarrow"]) # Try setting num cpus to 1 to avoid hangs? import pyarrow pyarrow.set_cpu_count(1) - return request.param + return parser @pytest.fixture(params=_c_parsers_only, ids=_c_parser_ids) @@ -119,7 +120,7 @@ def c_parser_only(request): """ Fixture all of the CSV parsers using the C engine. """ - return request.param + return request.param() @pytest.fixture(params=_py_parsers_only, ids=_py_parser_ids) @@ -127,7 +128,7 @@ def python_parser_only(request): """ Fixture all of the CSV parsers using the Python engine. """ - return request.param + return request.param() @pytest.fixture(params=_pyarrow_parsers_only, ids=_pyarrow_parsers_ids) @@ -135,7 +136,7 @@ def pyarrow_parser_only(request): """ Fixture all of the CSV parsers using the Pyarrow engine. """ - return request.param + return request.param() def _get_all_parser_float_precision_combinations(): @@ -147,7 +148,7 @@ def _get_all_parser_float_precision_combinations(): ids = [] for parser, parser_id in zip(_all_parsers, _all_parser_ids): for precision in parser.float_precision_choices: - params.append((parser, precision)) + params.append((parser(), precision)) ids.append(f"{parser_id}-{precision}") return {"params": params, "ids": ids} diff --git a/pandas/tests/io/parser/test_parse_dates.py b/pandas/tests/io/parser/test_parse_dates.py index 6ef6035b96fbe..a071fbe39320e 100644 --- a/pandas/tests/io/parser/test_parse_dates.py +++ b/pandas/tests/io/parser/test_parse_dates.py @@ -43,6 +43,7 @@ from pandas.io.parsers import read_csv xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail") +skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip") # constant _DEFAULT_DATETIME = datetime(1, 1, 1) @@ -1573,7 +1574,7 @@ def test_parse_timezone(all_parsers): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize( "date_string", ["32/32/2019", "02/30/2019", "13/13/2019", "13/2019", "a3/11/2018", "10/11/2o17"], @@ -1585,7 +1586,7 @@ def test_invalid_parse_delimited_date(all_parsers, date_string): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize( "date_string,dayfirst,expected", [ @@ -1608,7 +1609,7 @@ def test_parse_delimited_date_swap_no_warning( tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize( "date_string,dayfirst,expected", [ @@ -1643,6 +1644,8 @@ def _helper_hypothesis_delimited_date(call, date_string, **kwargs): return msg, result +@pytest.mark.slow +@skip_pyarrow @given(date_strategy) @settings(deadline=None) @pytest.mark.parametrize("delimiter", list(" -./")) @@ -1678,7 +1681,7 @@ def test_hypothesis_delimited_date(date_format, dayfirst, delimiter, test_dateti assert result == expected -@xfail_pyarrow +@skip_pyarrow @pytest.mark.parametrize( "names, usecols, parse_dates, missing_cols", [ @@ -1711,7 +1714,7 @@ def test_missing_parse_dates_column_raises( ) -@xfail_pyarrow +@skip_pyarrow def test_date_parser_and_names(all_parsers): # GH#33699 parser = all_parsers @@ -1721,7 +1724,7 @@ def test_date_parser_and_names(all_parsers): tm.assert_frame_equal(result, expected) -@xfail_pyarrow +@skip_pyarrow def test_date_parser_usecols_thousands(all_parsers): # GH#39365 data = """A,B,C diff --git a/pyproject.toml b/pyproject.toml index 03c1485bd4e35..7cf07c1d099ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ markers = [ "arm_slow: mark a test as slow for arm64 architecture", "arraymanager: mark a test to run with ArrayManager enabled", ] +log_cli = 1 [tool.mypy] platform = "linux-64" diff --git a/requirements-dev.txt b/requirements-dev.txt index 9b35de4bccb48..723d58aea208b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -41,6 +41,7 @@ pytest-cov pytest-xdist>=1.31 pytest-asyncio pytest-instafail +pytest-timeout seaborn statsmodels ipywidgets