From 22d91acd1f932ef87dbcdbe06d5c3b35bfd6d543 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 15 Feb 2022 18:59:06 -0800 Subject: [PATCH 1/6] CI: Don't run xdist if pytest.mark.single_cpu --- .circleci/config.yml | 2 +- .github/workflows/datamanger.yml | 2 +- .github/workflows/posix.yml | 4 ++-- .github/workflows/python-dev.yml | 2 +- azure-pipelines.yml | 2 +- ci/run_tests.sh | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc357101e79fd..0ff8aea3b545a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: resource_class: arm.medium environment: ENV_FILE: ci/deps/circle-38-arm64.yaml - PYTEST_WORKERS: auto + TEST_ARGS: "--numprocesses auto --dist=loadfile" PATTERN: "not slow and not network and not clipboard and not arm_slow" PYTEST_TARGET: "pandas" steps: diff --git a/.github/workflows/datamanger.yml b/.github/workflows/datamanger.yml index d355bdec82443..ef4ceaf30a921 100644 --- a/.github/workflows/datamanger.yml +++ b/.github/workflows/datamanger.yml @@ -47,7 +47,7 @@ jobs: env: PANDAS_DATA_MANAGER: array PATTERN: "not network and not clipboard and not single_cpu" - PYTEST_WORKERS: "auto" + TEST_ARGS: "--numprocesses auto --dist=loadfile" PYTEST_TARGET: pandas run: | source activate pandas-dev diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 35196ad2840c6..82fc34783ca17 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -57,8 +57,8 @@ jobs: LANG: ${{ matrix.lang || '' }} LC_ALL: ${{ matrix.lc_all || '' }} PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }} - TEST_ARGS: ${{ matrix.test_args || '' }} - PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }} + # Don't activate pytest-xdist (-n) if signle_cpu build + TEST_ARGS: ${{ format('{0} {1}', matrix.test_args || '', contains(matrix.pattern, 'not single_cpu') && '--numprocesses auto --dist=loadfile' || '' }} PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }} IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }} # TODO: re-enable coverage on pypy, its slow diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c287827206336..68869a3e74b60 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,7 +21,7 @@ on: - "doc/**" env: - PYTEST_WORKERS: "auto" + TEST_ARGS: "--numprocesses auto --dist=loadfile" PANDAS_CI: 1 PATTERN: "not slow and not network and not clipboard and not single_cpu" COVERAGE: true diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 747be0654ac70..ce7c8b30fcf3c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ pr: - 1.4.x variables: - PYTEST_WORKERS: auto + TEST_ARGS: "--numprocesses auto --dist=loadfile" PYTEST_TARGET: pandas PATTERN: "not slow and not high_memory and not db and not network and not single_cpu" PANDAS_CI: 1 diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 7c11eb13e0e1d..68cd64e71e15f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,7 +24,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -r fEs $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" @@ -39,7 +39,7 @@ sh -c "$PYTEST_CMD" if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array - PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" + PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -r fEs $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_AM_CMD="$PYTEST_AM_CMD -m \"$PATTERN and arraymanager\"" From 331312e18aa48645c954535279da48c0d8234f3e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 16 Feb 2022 10:32:02 -0800 Subject: [PATCH 2/6] Fix condidition --- .github/workflows/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index e600172ed1f04..fc7ff51d0e7b6 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -58,7 +58,7 @@ jobs: LC_ALL: ${{ matrix.lc_all || '' }} PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }} # Don't activate pytest-xdist (-n) if signle_cpu build - TEST_ARGS: ${{ format('{0} {1}', matrix.test_args || '', contains(matrix.pattern, 'not single_cpu') && '--numprocesses auto --dist=loadfile' || '' }} + TEST_ARGS: ${{ format('{0} {1}', matrix.test_args || '', contains(matrix.pattern, 'not single_cpu') && '--numprocesses auto --dist=loadfile' || '') }} PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }} IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }} # TODO: re-enable coverage on pypy, its slow From c010a0e01c2c3ee06076ead1137565b754e7f917 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 17 Feb 2022 19:34:22 -0800 Subject: [PATCH 3/6] Turn into a debug PR --- ci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 68cd64e71e15f..9d63a0b84fa47 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,7 +24,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -v -r fEs $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" From 540525a03b9105a35fd0c83e03ca22282ffc1fce Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Thu, 17 Feb 2022 22:34:01 -0800 Subject: [PATCH 4/6] Try seeing what pyarrow version we get if no mamba --- .github/workflows/posix.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index fc7ff51d0e7b6..1d6a097eb7332 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -125,7 +125,6 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: - mamba-version: "*" channels: conda-forge activate-environment: pandas-dev channel-priority: flexible From ddaee2a955b66ead6d5ed74cc1526009ba35016b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 18 Feb 2022 09:17:19 -0800 Subject: [PATCH 5/6] Trigger CI From b2fd1d09774f6cdab14fb57b776d245d4f3bf729 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 18 Feb 2022 13:51:02 -0800 Subject: [PATCH 6/6] reset to mamba to get v2, and remove setting cpu --- .github/workflows/posix.yml | 1 + pandas/tests/io/parser/conftest.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 1d6a097eb7332..fc7ff51d0e7b6 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -125,6 +125,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: + mamba-version: "*" channels: conda-forge activate-environment: pandas-dev channel-priority: flexible diff --git a/pandas/tests/io/parser/conftest.py b/pandas/tests/io/parser/conftest.py index 066f448d97505..1042c9a47f66d 100644 --- a/pandas/tests/io/parser/conftest.py +++ b/pandas/tests/io/parser/conftest.py @@ -110,9 +110,6 @@ def all_parsers(request): pytest.importorskip("pyarrow", VERSIONS["pyarrow"]) # Try finding a way to disable threads all together # for more stable CI runs - import pyarrow - - pyarrow.set_cpu_count(1) return parser