From 8910e3cfbc8451f98e44381b27894e66acd181e6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Sun, 13 Feb 2022 10:10:00 -0800 Subject: [PATCH] CI: Debug min build timing out (#45946) * CI: Debug min build timing out * ignore single tests * Fix mark * Fix another one * Change single -> single_cpu * Remove -v * Skip stateful dependent pyarrow failure * Address comments * Add importerror case * Set timeout too --- .github/workflows/datamanger.yml | 3 ++- .github/workflows/posix.yml | 19 ++++++++++--------- .github/workflows/python-dev.yml | 2 +- azure-pipelines.yml | 2 +- ci/azure/posix.yml | 12 ++++++------ pandas/compat/pyarrow.py | 2 ++ pandas/tests/base/test_unique.py | 2 +- pandas/tests/frame/methods/test_rank.py | 2 +- pandas/tests/io/excel/test_readers.py | 2 ++ pandas/tests/io/json/test_compression.py | 1 + pandas/tests/io/json/test_pandas.py | 2 ++ pandas/tests/io/parser/conftest.py | 2 +- pandas/tests/io/parser/test_network.py | 3 +++ pandas/tests/io/parser/test_parse_dates.py | 10 +++++++--- pandas/tests/io/pytables/test_append.py | 2 +- pandas/tests/io/pytables/test_categorical.py | 2 +- pandas/tests/io/pytables/test_errors.py | 2 +- .../tests/io/pytables/test_file_handling.py | 2 +- pandas/tests/io/pytables/test_keys.py | 2 +- pandas/tests/io/pytables/test_put.py | 2 +- pandas/tests/io/pytables/test_read.py | 2 +- .../io/pytables/test_retain_attributes.py | 2 +- pandas/tests/io/pytables/test_round_trip.py | 2 +- pandas/tests/io/pytables/test_select.py | 2 +- pandas/tests/io/pytables/test_store.py | 2 +- pandas/tests/io/pytables/test_time_series.py | 2 +- pandas/tests/io/test_clipboard.py | 2 +- pandas/tests/io/test_feather.py | 2 +- pandas/tests/io/test_fsspec.py | 3 +++ pandas/tests/io/test_parquet.py | 4 ++++ pandas/tests/io/test_sql.py | 10 ---------- pandas/tests/io/xml/test_to_xml.py | 1 + pandas/tests/series/methods/test_rank.py | 2 +- pandas/tests/test_algos.py | 2 +- pandas/tests/window/test_numba.py | 2 +- pandas/tests/window/test_online.py | 2 +- pyproject.toml | 2 +- test_fast.bat | 2 +- test_fast.sh | 2 +- 39 files changed, 69 insertions(+), 55 deletions(-) diff --git a/.github/workflows/datamanger.yml b/.github/workflows/datamanger.yml index 7e93696b3c317..63b7a88aa12c4 100644 --- a/.github/workflows/datamanger.yml +++ b/.github/workflows/datamanger.yml @@ -18,6 +18,7 @@ jobs: data_manager: name: Test experimental data manager runs-on: ubuntu-latest + timeout-minutes: 120 services: moto: image: motoserver/moto @@ -43,7 +44,7 @@ jobs: - name: Run tests env: PANDAS_DATA_MANAGER: array - PATTERN: "not network and not clipboard" + PATTERN: "not network and not clipboard and not single_cpu" PYTEST_WORKERS: "auto" PYTEST_TARGET: pandas run: | diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index b97ecff3e4cd6..0e8da7b66026f 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -22,18 +22,19 @@ jobs: defaults: run: shell: bash -l {0} + timeout-minutes: 120 strategy: matrix: settings: [ - [actions-38-downstream_compat.yaml, "not slow and not network", "", "", "", "", ""], - [actions-38-minimum_versions.yaml, "", "", "", "", "", ""], - [actions-38.yaml, "not slow and not network", "language-pack-it", "it_IT.utf8", "it_IT.utf8", "", ""], - [actions-38.yaml, "not slow and not network", "language-pack-zh-hans", "zh_CN.utf8", "zh_CN.utf8", "", ""], - [actions-38.yaml, "", "", "", "", "", ""], - [actions-pypy-38.yaml, "not slow", "", "", "", "", "--max-worker-restart 0"], - [actions-39.yaml, "", "", "", "", "", ""], - [actions-310-numpydev.yaml, "not slow and not network", "", "", "", "deprecate", "-W error"], - [actions-310.yaml, "", "", "", "", "", ""], + [actions-38-downstream_compat.yaml, "not slow and not network and not single_cpu", "", "", "", "", ""], + [actions-38-minimum_versions.yaml, "not single_cpu", "", "", "", "", ""], + [actions-38.yaml, "not slow and not network and not single_cpu", "language-pack-it", "it_IT.utf8", "it_IT.utf8", "", ""], + [actions-38.yaml, "not slow and not network and not single_cpu", "language-pack-zh-hans", "zh_CN.utf8", "zh_CN.utf8", "", ""], + [actions-38.yaml, "not single_cpu", "", "", "", "", ""], + [actions-pypy-38.yaml, "not slow and not single_cpu", "", "", "", "", "--max-worker-restart 0"], + [actions-39.yaml, "not single_cpu", "", "", "", "", ""], + [actions-310-numpydev.yaml, "not slow and not network and not single_cpu", "", "", "", "deprecate", "-W error"], + [actions-310.yaml, "not single_cpu", "", "", "", "", ""], ] fail-fast: false env: diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 988859ca4ac93..c287827206336 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -23,7 +23,7 @@ on: env: PYTEST_WORKERS: "auto" PANDAS_CI: 1 - PATTERN: "not slow and not network and not clipboard" + PATTERN: "not slow and not network and not clipboard and not single_cpu" COVERAGE: true PYTEST_TARGET: pandas diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 865c5709b724c..2a911fd5b5a0c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,7 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas - PATTERN: "not slow and not high_memory and not db and not network" + PATTERN: "not slow and not high_memory and not db and not network and not single_cpu" PANDAS_CI: 1 jobs: diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9002f35376613..e4a0d982223be 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -11,32 +11,32 @@ jobs: py38_macos_1: ENV_FILE: ci/deps/azure-macos-38.yaml CONDA_PY: "38" - PATTERN: "not slow" + PATTERN: "not slow and not single_cpu" PYTEST_TARGET: "pandas/tests/[a-h]*" py38_macos_2: ENV_FILE: ci/deps/azure-macos-38.yaml CONDA_PY: "38" - PATTERN: "not slow" + PATTERN: "not slow and not single_cpu" PYTEST_TARGET: "pandas/tests/[i-z]*" py39_macos_1: ENV_FILE: ci/deps/azure-macos-39.yaml CONDA_PY: "39" - PATTERN: "not slow" + PATTERN: "not slow and not single_cpu" PYTEST_TARGET: "pandas/tests/[a-h]*" py39_macos_2: ENV_FILE: ci/deps/azure-macos-39.yaml CONDA_PY: "39" - PATTERN: "not slow" + PATTERN: "not slow and not single_cpu" PYTEST_TARGET: "pandas/tests/[i-z]*" py310_macos_1: ENV_FILE: ci/deps/azure-macos-310.yaml CONDA_PY: "310" - PATTERN: "not slow" + PATTERN: "not slow and not single_cpu" PYTEST_TARGET: "pandas/tests/[a-h]*" py310_macos_2: ENV_FILE: ci/deps/azure-macos-310.yaml CONDA_PY: "310" - PATTERN: "not slow" + PATTERN: "not slow and not single_cpu" PYTEST_TARGET: "pandas/tests/[i-z]*" steps: diff --git a/pandas/compat/pyarrow.py b/pandas/compat/pyarrow.py index e6ac0c59e789a..00b205d018e89 100644 --- a/pandas/compat/pyarrow.py +++ b/pandas/compat/pyarrow.py @@ -13,6 +13,7 @@ pa_version_under4p0 = _palv < Version("4.0.0") pa_version_under5p0 = _palv < Version("5.0.0") pa_version_under6p0 = _palv < Version("6.0.0") + pa_version_under7p0 = _palv < Version("7.0.0") except ImportError: pa_version_under1p01 = True pa_version_under2p0 = True @@ -20,3 +21,4 @@ pa_version_under4p0 = True pa_version_under5p0 = True pa_version_under6p0 = True + pa_version_under7p0 = True diff --git a/pandas/tests/base/test_unique.py b/pandas/tests/base/test_unique.py index e399ae45160fc..66cc000b9f458 100644 --- a/pandas/tests/base/test_unique.py +++ b/pandas/tests/base/test_unique.py @@ -104,7 +104,7 @@ def test_nunique_null(null_obj, index_or_series_obj): assert obj.nunique(dropna=False) == max(0, num_unique_values) -@pytest.mark.single +@pytest.mark.single_cpu @pytest.mark.xfail( reason="Flaky in the CI. Remove once CI has a single build: GH 44584", strict=False ) diff --git a/pandas/tests/frame/methods/test_rank.py b/pandas/tests/frame/methods/test_rank.py index 48188b66c45b5..31a6d7637a244 100644 --- a/pandas/tests/frame/methods/test_rank.py +++ b/pandas/tests/frame/methods/test_rank.py @@ -327,7 +327,7 @@ def test_rank_pct_true(self, method, exp): expected = DataFrame(exp) tm.assert_frame_equal(result, expected) - @pytest.mark.single + @pytest.mark.single_cpu @pytest.mark.high_memory def test_pct_max_many_rows(self): # GH 18271 diff --git a/pandas/tests/io/excel/test_readers.py b/pandas/tests/io/excel/test_readers.py index 6d533d654369c..5fe1b5fec4ba2 100644 --- a/pandas/tests/io/excel/test_readers.py +++ b/pandas/tests/io/excel/test_readers.py @@ -775,6 +775,7 @@ def test_read_from_http_url(self, read_ext): tm.assert_frame_equal(url_table, local_table) @td.skip_if_not_us_locale + @pytest.mark.single_cpu def test_read_from_s3_url(self, read_ext, s3_resource, s3so): # Bucket "pandas-test" created in tests/io/conftest.py with open("test1" + read_ext, "rb") as f: @@ -786,6 +787,7 @@ def test_read_from_s3_url(self, read_ext, s3_resource, s3so): local_table = pd.read_excel("test1" + read_ext) tm.assert_frame_equal(url_table, local_table) + @pytest.mark.single_cpu def test_read_from_s3_object(self, read_ext, s3_resource, s3so): # GH 38788 # Bucket "pandas-test" created in tests/io/conftest.py diff --git a/pandas/tests/io/json/test_compression.py b/pandas/tests/io/json/test_compression.py index a752c93a8046a..42e7b6cd03f55 100644 --- a/pandas/tests/io/json/test_compression.py +++ b/pandas/tests/io/json/test_compression.py @@ -38,6 +38,7 @@ def test_read_zipped_json(datapath): @td.skip_if_not_us_locale +@pytest.mark.single_cpu def test_with_s3_url(compression, s3_resource, s3so): # Bucket "pandas-test" created in tests/io/conftest.py diff --git a/pandas/tests/io/json/test_pandas.py b/pandas/tests/io/json/test_pandas.py index fd523d938c784..47ae7221b4c92 100644 --- a/pandas/tests/io/json/test_pandas.py +++ b/pandas/tests/io/json/test_pandas.py @@ -1222,6 +1222,7 @@ def test_read_inline_jsonl(self): expected = DataFrame([[1, 2], [1, 2]], columns=["a", "b"]) tm.assert_frame_equal(result, expected) + @pytest.mark.single_cpu @td.skip_if_not_us_locale def test_read_s3_jsonl(self, s3_resource, s3so): # GH17200 @@ -1749,6 +1750,7 @@ def test_json_multiindex(self, dataframe, expected): result = series.to_json(orient="index") assert result == expected + @pytest.mark.single_cpu def test_to_s3(self, s3_resource, s3so): import time diff --git a/pandas/tests/io/parser/conftest.py b/pandas/tests/io/parser/conftest.py index b2d2be362d0d3..7428432aed6e5 100644 --- a/pandas/tests/io/parser/conftest.py +++ b/pandas/tests/io/parser/conftest.py @@ -110,7 +110,7 @@ def all_parsers(request): pytest.importorskip("pyarrow", VERSIONS["pyarrow"]) # Try setting num cpus to 1 to avoid hangs on Azure MacOS/Windows builds # or better yet find a way to disable threads - # TODO(GH#44584) pytest.mark.single these tests + # TODO(GH#44584) pytest.mark.single_cpu these tests import pyarrow pyarrow.set_cpu_count(1) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index f5470b80a938a..1dcafe8ac45e8 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -66,6 +66,7 @@ def tips_df(datapath): return read_csv(datapath("io", "data", "csv", "tips.csv")) +@pytest.mark.single_cpu @pytest.mark.usefixtures("s3_resource") @td.skip_if_not_us_locale() class TestS3: @@ -242,6 +243,7 @@ def test_write_s3_parquet_fails(self, tips_df, s3so): storage_options=s3so, ) + @pytest.mark.single_cpu def test_read_csv_handles_boto_s3_object(self, s3_resource, tips_file): # see gh-16135 @@ -257,6 +259,7 @@ def test_read_csv_handles_boto_s3_object(self, s3_resource, tips_file): expected = read_csv(tips_file) tm.assert_frame_equal(result, expected) + @pytest.mark.single_cpu @pytest.mark.skipif( is_ci_environment(), reason="This test can hang in our CI min_versions build " diff --git a/pandas/tests/io/parser/test_parse_dates.py b/pandas/tests/io/parser/test_parse_dates.py index 4777bed2a2460..fe5857383ee2b 100644 --- a/pandas/tests/io/parser/test_parse_dates.py +++ b/pandas/tests/io/parser/test_parse_dates.py @@ -18,7 +18,10 @@ from pandas._libs.tslibs import parsing from pandas._libs.tslibs.parsing import parse_datetime_string -from pandas.compat.pyarrow import pa_version_under6p0 +from pandas.compat.pyarrow import ( + pa_version_under6p0, + pa_version_under7p0, +) import pandas as pd from pandas import ( @@ -948,10 +951,11 @@ def test_parse_dates_custom_euro_format(all_parsers, kwargs): ) -@xfail_pyarrow -def test_parse_tz_aware(all_parsers): +def test_parse_tz_aware(all_parsers, request): # See gh-1693 parser = all_parsers + if parser.engine == "pyarrow" and pa_version_under7p0: + request.node.add_marker(pytest.mark.xfail(reason="Fails for pyarrow < 7.0")) data = "Date,x\n2012-06-13T01:39:00Z,0.5" result = parser.read_csv(StringIO(data), index_col=0, parse_dates=True) diff --git a/pandas/tests/io/pytables/test_append.py b/pandas/tests/io/pytables/test_append.py index 5544b8112627b..f38ea4d5cf306 100644 --- a/pandas/tests/io/pytables/test_append.py +++ b/pandas/tests/io/pytables/test_append.py @@ -24,7 +24,7 @@ ensure_clean_store, ) -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu @pytest.mark.filterwarnings("ignore:object name:tables.exceptions.NaturalNameWarning") diff --git a/pandas/tests/io/pytables/test_categorical.py b/pandas/tests/io/pytables/test_categorical.py index dee373000b7d5..3ef685d1132b1 100644 --- a/pandas/tests/io/pytables/test_categorical.py +++ b/pandas/tests/io/pytables/test_categorical.py @@ -16,7 +16,7 @@ ) pytestmark = [ - pytest.mark.single, + pytest.mark.single_cpu, # pytables https://github.com/PyTables/PyTables/issues/822 pytest.mark.filterwarnings( "ignore:a closed node found in the registry:UserWarning" diff --git a/pandas/tests/io/pytables/test_errors.py b/pandas/tests/io/pytables/test_errors.py index dbcd112a28aca..f31c185af0497 100644 --- a/pandas/tests/io/pytables/test_errors.py +++ b/pandas/tests/io/pytables/test_errors.py @@ -25,7 +25,7 @@ _maybe_adjust_name, ) -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_pass_spec_to_storer(setup_path): diff --git a/pandas/tests/io/pytables/test_file_handling.py b/pandas/tests/io/pytables/test_file_handling.py index 027c3d0bd821c..9fde65e3a1a43 100644 --- a/pandas/tests/io/pytables/test_file_handling.py +++ b/pandas/tests/io/pytables/test_file_handling.py @@ -26,7 +26,7 @@ Term, ) -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu @pytest.mark.parametrize("mode", ["r", "r+", "a", "w"]) diff --git a/pandas/tests/io/pytables/test_keys.py b/pandas/tests/io/pytables/test_keys.py index 02b79bd0fdbc1..cf412a242b000 100644 --- a/pandas/tests/io/pytables/test_keys.py +++ b/pandas/tests/io/pytables/test_keys.py @@ -11,7 +11,7 @@ tables, ) -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_keys(setup_path): diff --git a/pandas/tests/io/pytables/test_put.py b/pandas/tests/io/pytables/test_put.py index 8e7b31bcf8bca..d6ef6e5d9fe0f 100644 --- a/pandas/tests/io/pytables/test_put.py +++ b/pandas/tests/io/pytables/test_put.py @@ -29,7 +29,7 @@ ) from pandas.util import _test_decorators as td -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_format_type(setup_path): diff --git a/pandas/tests/io/pytables/test_read.py b/pandas/tests/io/pytables/test_read.py index 1c9e63c66aadb..2374ba345eeb5 100644 --- a/pandas/tests/io/pytables/test_read.py +++ b/pandas/tests/io/pytables/test_read.py @@ -25,7 +25,7 @@ from pandas.io.pytables import TableIterator -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_read_missing_key_close_store(setup_path): diff --git a/pandas/tests/io/pytables/test_retain_attributes.py b/pandas/tests/io/pytables/test_retain_attributes.py index 16772d03c6d26..bd02121bd3252 100644 --- a/pandas/tests/io/pytables/test_retain_attributes.py +++ b/pandas/tests/io/pytables/test_retain_attributes.py @@ -17,7 +17,7 @@ ensure_clean_store, ) -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_retain_index_attributes(setup_path): diff --git a/pandas/tests/io/pytables/test_round_trip.py b/pandas/tests/io/pytables/test_round_trip.py index 867525a7146b4..689bdab54d11f 100644 --- a/pandas/tests/io/pytables/test_round_trip.py +++ b/pandas/tests/io/pytables/test_round_trip.py @@ -30,7 +30,7 @@ _default_compressor = "blosc" -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_conv_read_write(setup_path): diff --git a/pandas/tests/io/pytables/test_select.py b/pandas/tests/io/pytables/test_select.py index b644c3420150c..b9bee8d485f41 100644 --- a/pandas/tests/io/pytables/test_select.py +++ b/pandas/tests/io/pytables/test_select.py @@ -27,7 +27,7 @@ from pandas.io.pytables import Term -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_select_columns_in_where(setup_path): diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index cbca8bb64e350..72e25d31d943f 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -40,7 +40,7 @@ read_hdf, ) -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_context(setup_path): diff --git a/pandas/tests/io/pytables/test_time_series.py b/pandas/tests/io/pytables/test_time_series.py index 5e42dbde4b9f1..6625984961c11 100644 --- a/pandas/tests/io/pytables/test_time_series.py +++ b/pandas/tests/io/pytables/test_time_series.py @@ -10,7 +10,7 @@ ) from pandas.tests.io.pytables.common import ensure_clean_store -pytestmark = pytest.mark.single +pytestmark = pytest.mark.single_cpu def test_store_datetime_fractional_secs(setup_path): diff --git a/pandas/tests/io/test_clipboard.py b/pandas/tests/io/test_clipboard.py index aacfccd38b7da..0cd4f9c02f69f 100644 --- a/pandas/tests/io/test_clipboard.py +++ b/pandas/tests/io/test_clipboard.py @@ -148,7 +148,7 @@ def test_mock_clipboard(mock_clipboard): assert result == "abc" -@pytest.mark.single +@pytest.mark.single_cpu @pytest.mark.clipboard @pytest.mark.usefixtures("mock_clipboard") class TestClipboard: diff --git a/pandas/tests/io/test_feather.py b/pandas/tests/io/test_feather.py index bc1a5266001c8..7c2ce37d8aa70 100644 --- a/pandas/tests/io/test_feather.py +++ b/pandas/tests/io/test_feather.py @@ -14,7 +14,7 @@ @filter_sparse -@pytest.mark.single +@pytest.mark.single_cpu @pytest.mark.filterwarnings("ignore:CategoricalBlock is deprecated:DeprecationWarning") class TestFeather: def check_error_on_write(self, df, exc, err_msg): diff --git a/pandas/tests/io/test_fsspec.py b/pandas/tests/io/test_fsspec.py index f1040c0bd30f2..8f991cfaadf8b 100644 --- a/pandas/tests/io/test_fsspec.py +++ b/pandas/tests/io/test_fsspec.py @@ -199,6 +199,7 @@ def test_fastparquet_options(fsspectest): assert fsspectest.test[0] == "parquet_read" +@pytest.mark.single_cpu @td.skip_if_no("s3fs") def test_from_s3_csv(s3_resource, tips_file, s3so): tm.assert_equal( @@ -215,6 +216,7 @@ def test_from_s3_csv(s3_resource, tips_file, s3so): ) +@pytest.mark.single_cpu @pytest.mark.parametrize("protocol", ["s3", "s3a", "s3n"]) @td.skip_if_no("s3fs") def test_s3_protocols(s3_resource, tips_file, protocol, s3so): @@ -224,6 +226,7 @@ def test_s3_protocols(s3_resource, tips_file, protocol, s3so): ) +@pytest.mark.single_cpu @td.skip_array_manager_not_yet_implemented # TODO(ArrayManager) fastparquet @td.skip_if_no("s3fs") @td.skip_if_no("fastparquet") diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index e6dbcefcc6842..a5973af3799eb 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -741,6 +741,7 @@ def test_categorical(self, pa): check_round_trip(df, pa) + @pytest.mark.single_cpu def test_s3_roundtrip_explicit_fs(self, df_compat, s3_resource, pa, s3so): s3fs = pytest.importorskip("s3fs") s3 = s3fs.S3FileSystem(**s3so) @@ -753,6 +754,7 @@ def test_s3_roundtrip_explicit_fs(self, df_compat, s3_resource, pa, s3so): write_kwargs=kw, ) + @pytest.mark.single_cpu def test_s3_roundtrip(self, df_compat, s3_resource, pa, s3so): # GH #19134 s3so = {"storage_options": s3so} @@ -764,6 +766,7 @@ def test_s3_roundtrip(self, df_compat, s3_resource, pa, s3so): write_kwargs=s3so, ) + @pytest.mark.single_cpu @td.skip_if_no("s3fs") # also requires flask @pytest.mark.parametrize( "partition_col", @@ -1006,6 +1009,7 @@ def test_filter_row_groups(self, fp): result = read_parquet(path, fp, filters=[("a", "==", 0)]) assert len(result) == 1 + @pytest.mark.single_cpu def test_s3_roundtrip(self, df_compat, s3_resource, fp, s3so): # GH #19134 check_round_trip( diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 17314959f058a..8263c437030eb 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -1277,7 +1277,6 @@ def test_escaped_table_name(self): tm.assert_frame_equal(res, df) -@pytest.mark.single @pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="SQLAlchemy not installed") class TestSQLApi(SQLAlchemyMixIn, _TestSQLApi): """ @@ -1497,12 +1496,10 @@ def setup_method(self, load_iris_data, load_types_data): self.pandasSQL = sql.SQLDatabase(self.__engine) -@pytest.mark.single class TestSQLApiConn(_EngineToConnMixin, TestSQLApi): pass -@pytest.mark.single class TestSQLiteFallbackApi(SQLiteMixIn, _TestSQLApi): """ Test the public sqlite connection fallback API @@ -2540,36 +2537,30 @@ def test_schema_support(self): tm.assert_frame_equal(res1, res2) -@pytest.mark.single @pytest.mark.db class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy): pass -@pytest.mark.single @pytest.mark.db class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn): pass -@pytest.mark.single @pytest.mark.db class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy): pass -@pytest.mark.single @pytest.mark.db class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn): pass -@pytest.mark.single class TestSQLiteAlchemy(_TestSQLiteAlchemy, _TestSQLAlchemy): pass -@pytest.mark.single class TestSQLiteAlchemyConn(_TestSQLiteAlchemy, _TestSQLAlchemyConn): pass @@ -2578,7 +2569,6 @@ class TestSQLiteAlchemyConn(_TestSQLiteAlchemy, _TestSQLAlchemyConn): # -- Test Sqlite / MySQL fallback -@pytest.mark.single class TestSQLiteFallback(SQLiteMixIn, PandasSQLTest): """ Test the fallback mode against an in-memory sqlite database. diff --git a/pandas/tests/io/xml/test_to_xml.py b/pandas/tests/io/xml/test_to_xml.py index beaa6d61f02c2..5859592ad7bf2 100644 --- a/pandas/tests/io/xml/test_to_xml.py +++ b/pandas/tests/io/xml/test_to_xml.py @@ -1331,6 +1331,7 @@ def test_unsuported_compression(datapath, parser): # STORAGE OPTIONS +@pytest.mark.single_cpu @td.skip_if_no("s3fs") @td.skip_if_no("lxml") def test_s3_permission_output(parser, s3_resource): diff --git a/pandas/tests/series/methods/test_rank.py b/pandas/tests/series/methods/test_rank.py index d85b84bec55ac..b49ffda4a7a91 100644 --- a/pandas/tests/series/methods/test_rank.py +++ b/pandas/tests/series/methods/test_rank.py @@ -475,7 +475,7 @@ def test_rank_first_pct(dtype, ser, exp): tm.assert_series_equal(result, expected) -@pytest.mark.single +@pytest.mark.single_cpu @pytest.mark.high_memory def test_pct_max_many_rows(): # GH 18271 diff --git a/pandas/tests/test_algos.py b/pandas/tests/test_algos.py index 94a20901b2f7a..0007a966840e8 100644 --- a/pandas/tests/test_algos.py +++ b/pandas/tests/test_algos.py @@ -1787,7 +1787,7 @@ def test_too_many_ndims(self): with pytest.raises(TypeError, match=msg): algos.rank(arr) - @pytest.mark.single + @pytest.mark.single_cpu @pytest.mark.high_memory def test_pct_max_many_rows(self): # GH 18271 diff --git a/pandas/tests/window/test_numba.py b/pandas/tests/window/test_numba.py index be6eade37a341..9fe7ae7a5bb90 100644 --- a/pandas/tests/window/test_numba.py +++ b/pandas/tests/window/test_numba.py @@ -18,7 +18,7 @@ import pandas._testing as tm from pandas.core.util.numba_ import NUMBA_FUNC_CACHE -# TODO(GH#44584): Mark these as pytest.mark.single +# TODO(GH#44584): Mark these as pytest.mark.single_cpu pytestmark = pytest.mark.skipif( is_ci_environment() and (is_platform_windows() or is_platform_mac()), reason="On Azure CI, Windows can fail with " diff --git a/pandas/tests/window/test_online.py b/pandas/tests/window/test_online.py index c157b0e082405..ab435a39a497b 100644 --- a/pandas/tests/window/test_online.py +++ b/pandas/tests/window/test_online.py @@ -14,7 +14,7 @@ ) import pandas._testing as tm -# TODO(GH#44584): Mark these as pytest.mark.single +# TODO(GH#44584): Mark these as pytest.mark.single_cpu pytestmark = pytest.mark.skipif( is_ci_environment() and (is_platform_windows() or is_platform_mac()), reason="On Azure CI, Windows can fail with " diff --git a/pyproject.toml b/pyproject.toml index 4455df705fc3f..1d318ab5f70c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ filterwarnings = [ ] junit_family = "xunit2" markers = [ - "single: mark a test as single cpu only", + "single_cpu: mark a test that should run on a single cpu only", "slow: mark a test as slow", "network: mark a test as network", "db: tests requiring a database (mysql or postgres)", diff --git a/test_fast.bat b/test_fast.bat index 642e0549f3228..85ce305ab2d64 100644 --- a/test_fast.bat +++ b/test_fast.bat @@ -1,3 +1,3 @@ :: test on windows set PYTHONHASHSEED=314159265 -pytest --skip-slow --skip-network --skip-db -m "not single" -n 4 -r sXX pandas +pytest --skip-slow --skip-network --skip-db -m "not single_cpu" -n 4 -r sXX pandas diff --git a/test_fast.sh b/test_fast.sh index 9d446964cf501..0120c712e3df8 100755 --- a/test_fast.sh +++ b/test_fast.sh @@ -5,4 +5,4 @@ # https://github.com/pytest-dev/pytest/issues/1075 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') -pytest pandas --skip-slow --skip-network --skip-db -m "not single" -n 4 -r sxX "$@" +pytest pandas --skip-slow --skip-network --skip-db -m "not single_cpu" -n 4 -r sxX "$@"