From b3e63463802b36a03f4232996eec80f0cec2006a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 18:06:29 -0800 Subject: [PATCH 1/6] TST: merge high_memory mark with single_cpu --- azure-pipelines.yml | 2 +- pandas/conftest.py | 8 -------- pandas/tests/frame/methods/test_rank.py | 1 - pandas/tests/io/parser/test_c_parser_only.py | 4 ++-- pandas/tests/series/methods/test_rank.py | 1 - pandas/tests/test_algos.py | 1 - pyproject.toml | 3 +-- 7 files changed, 4 insertions(+), 16 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dc0714871503c..bc36a00669656 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,7 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas - PATTERN: "not slow and not high_memory and not db and not network and not single_cpu" + PATTERN: "not slow and and not db and not network and not single_cpu" PANDAS_CI: 1 jobs: diff --git a/pandas/conftest.py b/pandas/conftest.py index 8d5913ce0a9ae..0b79830c1bcf3 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -106,7 +106,6 @@ def pytest_collection_modifyitems(items, config): only_slow = config.getoption("--only-slow") skip_network = config.getoption("--skip-network") skip_db = config.getoption("--skip-db") - run_high_memory = config.getoption("--run-high-memory") marks = [ (pytest.mark.slow, "slow", skip_slow, "--skip-slow"), @@ -139,13 +138,6 @@ def pytest_collection_modifyitems(items, config): if only_slow and "slow" not in item.keywords: item.add_marker(pytest.mark.skip("skipping due to --only-slow")) - if "high_memory" in item.keywords and not run_high_memory: - item.add_marker( - pytest.mark.skip( - "skipping high memory test since --run-high-memory was not set" - ) - ) - # Hypothesis hypothesis.settings.register_profile( diff --git a/pandas/tests/frame/methods/test_rank.py b/pandas/tests/frame/methods/test_rank.py index 68c87a59d8230..7b2f7908673e3 100644 --- a/pandas/tests/frame/methods/test_rank.py +++ b/pandas/tests/frame/methods/test_rank.py @@ -328,7 +328,6 @@ def test_rank_pct_true(self, method, exp): tm.assert_frame_equal(result, expected) @pytest.mark.single_cpu - @pytest.mark.high_memory def test_pct_max_many_rows(self): # GH 18271 df = DataFrame( diff --git a/pandas/tests/io/parser/test_c_parser_only.py b/pandas/tests/io/parser/test_c_parser_only.py index 83cccdb37b343..a0819da04c4b0 100644 --- a/pandas/tests/io/parser/test_c_parser_only.py +++ b/pandas/tests/io/parser/test_c_parser_only.py @@ -555,7 +555,7 @@ def test_read_tarfile(c_parser_only, csv_dir_path, tar_suffix): tm.assert_frame_equal(out, expected) -@pytest.mark.high_memory +@pytest.mark.single_cpu def test_bytes_exceed_2gb(c_parser_only): # see gh-16798 # @@ -563,7 +563,7 @@ def test_bytes_exceed_2gb(c_parser_only): parser = c_parser_only if parser.low_memory: - pytest.skip("not a high_memory test") + pytest.skip("not a low_memory test") csv = StringIO("strings\n" + "\n".join(["x" * (1 << 20) for _ in range(2100)])) df = parser.read_csv(csv) diff --git a/pandas/tests/series/methods/test_rank.py b/pandas/tests/series/methods/test_rank.py index 4ab99673dfe46..3af06502a3066 100644 --- a/pandas/tests/series/methods/test_rank.py +++ b/pandas/tests/series/methods/test_rank.py @@ -476,7 +476,6 @@ def test_rank_first_pct(dtype, ser, exp): @pytest.mark.single_cpu -@pytest.mark.high_memory def test_pct_max_many_rows(): # GH 18271 s = Series(np.arange(2**24 + 1)) diff --git a/pandas/tests/test_algos.py b/pandas/tests/test_algos.py index db09c15cd136b..7c16d13e59d3a 100644 --- a/pandas/tests/test_algos.py +++ b/pandas/tests/test_algos.py @@ -1659,7 +1659,6 @@ def test_too_many_ndims(self): algos.rank(arr) @pytest.mark.single_cpu - @pytest.mark.high_memory def test_pct_max_many_rows(self): # GH 18271 values = np.arange(2**24 + 1) diff --git a/pyproject.toml b/pyproject.toml index 19fd5335bfa9a..ae4072d37a22d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,11 +48,10 @@ filterwarnings = [ ] junit_family = "xunit2" markers = [ - "single_cpu: mark a test that should run on a single cpu only", + "single_cpu: tests that should run on a single cpu only", "slow: mark a test as slow", "network: mark a test as network", "db: tests requiring a database (mysql or postgres)", - "high_memory: mark a test as a high-memory only", "clipboard: mark a pd.read_clipboard test", "arm_slow: mark a test as slow for arm64 architecture", "arraymanager: mark a test to run with ArrayManager enabled", From cea3ffd08a55c244c6568343ebc9657b2b5993f7 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 20:24:20 -0800 Subject: [PATCH 2/6] Skip a test --- pandas/tests/io/parser/test_c_parser_only.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/tests/io/parser/test_c_parser_only.py b/pandas/tests/io/parser/test_c_parser_only.py index a0819da04c4b0..71af12b10b417 100644 --- a/pandas/tests/io/parser/test_c_parser_only.py +++ b/pandas/tests/io/parser/test_c_parser_only.py @@ -17,7 +17,10 @@ import numpy as np import pytest -from pandas.compat import IS64 +from pandas.compat import ( + IS64, + is_ci_environment, +) from pandas.errors import ParserError import pandas.util._test_decorators as td @@ -556,6 +559,7 @@ def test_read_tarfile(c_parser_only, csv_dir_path, tar_suffix): @pytest.mark.single_cpu +@pytest.mark.skipif(is_ci_environment(), reason="Too memory intensive for CI.") def test_bytes_exceed_2gb(c_parser_only): # see gh-16798 # From 500612d5b607e3976674c99f7fee356190cad85c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 20:25:09 -0800 Subject: [PATCH 3/6] Remove extra and --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bc36a00669656..2d5a0b0a6c279 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,7 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas - PATTERN: "not slow and and not db and not network and not single_cpu" + PATTERN: "not slow and not db and not network and not single_cpu" PANDAS_CI: 1 jobs: From bf167a13efbc56584dd77a44128cece4ea98e6a3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 21:44:23 -0800 Subject: [PATCH 4/6] Ensure PANDAS_CI is set during docker run --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2d5a0b0a6c279..29a9c13ce2cf7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,7 +41,7 @@ jobs: - script: | docker pull quay.io/pypa/manylinux2014_i686 docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ + PANDAS_CI="1" /bin/bash -xc "cd pandas && \ /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ From 067ac2aceb1c73404edf1e0e368cca4f473fe7e7 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Mon, 7 Mar 2022 21:46:40 -0800 Subject: [PATCH 5/6] Ensure PANDAS_CI is set in circle CI --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc357101e79fd..a16492b022264 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,7 @@ jobs: PYTEST_WORKERS: auto PATTERN: "not slow and not network and not clipboard and not arm_slow" PYTEST_TARGET: "pandas" + PANDAS_CI: "1" steps: - checkout - run: ci/setup_env.sh From 3f1efd56fcf11e1f28482dd783b3f154c2e9bd28 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 8 Mar 2022 09:48:03 -0800 Subject: [PATCH 6/6] Don't run single_cpu tests on docker 32 bit --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 29a9c13ce2cf7..e6dc98179b7d6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,14 +41,14 @@ jobs: - script: | docker pull quay.io/pypa/manylinux2014_i686 docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - PANDAS_CI="1" /bin/bash -xc "cd pandas && \ + /bin/bash -xc "cd pandas && \ /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ python setup.py build_ext -q -j2 && \ python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" + pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - task: PublishTestResults@2