CI: Add single CPU core build for pytest.mark.single

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     environment:
       ENV_FILE: ci/deps/circle-38-arm64.yaml
       PYTEST_WORKERS: auto
-      PATTERN: "not slow and not network and not clipboard and not arm_slow"
+      PATTERN: "not slow and not network and not clipboard and not arm_slow and not single"
       PYTEST_TARGET: "pandas"
     steps:
       - checkout

.github/workflows/database.yml

@@ -15,7 +15,7 @@ on:
 env:
   PYTEST_WORKERS: "auto"
   PANDAS_CI: 1
-  PATTERN: ((not slow and not network and not clipboard) or (single and db))
+  PATTERN: db
   COVERAGE: true
 
 jobs:

.github/workflows/posix.yml

@@ -13,7 +13,6 @@ on:
       - "doc/**"
 
 env:
-  PYTEST_WORKERS: "auto"
   PANDAS_CI: 1
 
 jobs:
@@ -25,14 +24,16 @@ jobs:
     strategy:
       matrix:
         settings: [
-          [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-38-slow.yaml, "slow", "", "", "", "", ""],
-          [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-39-slow.yaml, "slow", "", "", "", "", ""],
-          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not network and not clipboard", "", "", "", "", ""]
+          [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard and not single", "", "", "", "", "", "auto"],
+          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", "", "auto"],
+          [actions-38.yaml, "not slow and not network and not clipboard and not single", "", "", "", "", "", "auto"],
+          [actions-38-slow-single.yaml, "slow", "", "", "", "", "", "auto"],
+          [actions-38-slow-single.yaml, "single", "", "", "", "", "", "1"],
+          [actions-38-locale.yaml, "not slow and not network and not single", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", "", "auto"],
+          [actions-39-slow-single.yaml, "slow", "", "", "", "", "", "auto"],
+          [actions-39-slow-single.yaml, "single", "", "", "", "", "", "1"],
+          [actions-39-numpydev.yaml, "not slow and not network and not single", "xsel", "", "", "deprecate", "-W error", "auto"],
+          [actions-39.yaml, "not slow and not network and not clipboard and not single", "", "", "", "", "", "auto"]
         ]
       fail-fast: false
     env:
@@ -44,6 +45,7 @@ jobs:
       LC_ALL: ${{ matrix.settings[4] }}
       PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
       TEST_ARGS: ${{ matrix.settings[6] }}
+      PYTEST_WORKERS: ${{ matrix.settings[7] }}
       PYTEST_TARGET:  pandas
     concurrency:
       group: ${{ github.ref }}-${{ matrix.settings[0] }}

ci/azure/posix.yml

@@ -9,15 +9,20 @@ jobs:
   strategy:
     matrix:
       ${{ if eq(parameters.name, 'macOS') }}:
+        py38_macos_single:
+          ENV_FILE: ci/deps/azure-macos-38.yaml
+          CONDA_PY: "38"
+          PATTERN: "single"
+          PYTEST_WORKERS: 1
         py38_macos_1:
           ENV_FILE: ci/deps/azure-macos-38.yaml
           CONDA_PY: "38"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not single"
           PYTEST_TARGET: "pandas/tests/[a-h]*"
         py38_macos_2:
           ENV_FILE: ci/deps/azure-macos-38.yaml
           CONDA_PY: "38"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not single"
           PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:

ci/azure/windows.yml

@@ -8,31 +8,43 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
+      py38_np18_single:
+        ENV_FILE: ci/deps/azure-windows-38.yaml
+        CONDA_PY: "38"
+        PATTERN: "single"
+        PYTEST_WORKERS: 1
+
+      py39_single:
+        ENV_FILE: ci/deps/azure-windows-39.yaml
+        CONDA_PY: "39"
+        PATTERN: "single"
+        PYTEST_WORKERS: 1
+
       py38_np18_1:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow and not network"
+        PATTERN: "not slow and not network and not single"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[a-h]*"
 
       py38_np18_2:
         ENV_FILE: ci/deps/azure-windows-38.yaml
         CONDA_PY: "38"
-        PATTERN: "not slow and not network"
+        PATTERN: "not slow and not network and not single"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[i-z]*"
 
       py39_1:
         ENV_FILE: ci/deps/azure-windows-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow and not network and not high_memory"
+        PATTERN: "not slow and not network and not single"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[a-h]*"
 
       py39_2:
         ENV_FILE: ci/deps/azure-windows-39.yaml
         CONDA_PY: "39"
-        PATTERN: "not slow and not network and not high_memory"
+        PATTERN: "not slow and not network and not single"
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[i-z]*"
 

ci/run_tests.sh

@@ -19,7 +19,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="${XVFB}pytest -v -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ $(uname) != "Linux"  && $(uname) != "Darwin" ]]; then
     PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/"

pandas/conftest.py

@@ -98,7 +98,6 @@ def pytest_collection_modifyitems(items, config):
     only_slow = config.getoption("--only-slow")
     skip_network = config.getoption("--skip-network")
     skip_db = config.getoption("--skip-db")
-    run_high_memory = config.getoption("--run-high-memory")
 
     marks = [
         (pytest.mark.slow, "slow", skip_slow, "--skip-slow"),
@@ -125,13 +124,6 @@ def pytest_collection_modifyitems(items, config):
         if only_slow and "slow" not in item.keywords:
             item.add_marker(pytest.mark.skip("skipping due to --only-slow"))
 
-        if "high_memory" in item.keywords and not run_high_memory:
-            item.add_marker(
-                pytest.mark.skip(
-                    "skipping high memory test since --run-high-memory was not set"
-                )
-            )
-
 
 # Hypothesis
 hypothesis.settings.register_profile(

pandas/tests/frame/methods/test_rank.py

@@ -325,7 +325,6 @@ def test_rank_pct_true(self, method, exp):
         tm.assert_frame_equal(result, expected)
 
     @pytest.mark.single
-    @pytest.mark.high_memory
     def test_pct_max_many_rows(self):
         # GH 18271
         df = DataFrame(

pandas/tests/io/conftest.py

@@ -163,3 +163,11 @@ def add_tips_files(bucket_name):
     while cli.list_buckets()["Buckets"] and timeout > 0:
         time.sleep(0.1)
         timeout -= 0.1
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        # All tests that use the s3_resource should be run with 1 cpu core in the CI
+        # to (hopefully) reduce flakiness (i.e. moto_server timeout) in test execution
+        if "s3_resource" in getattr(item, "fixturenames", ()):
+            item.add_marker(pytest.mark.single)

pandas/tests/io/parser/test_c_parser_only.py

@@ -555,7 +555,9 @@ def test_read_tarfile(c_parser_only, csv_dir_path, tar_suffix):
         tm.assert_frame_equal(out, expected)
 
 
-@pytest.mark.high_memory
+@pytest.mark.single
+@td.skip_if_windows  # CI: MemoryError
+@td.skip_if_32bit  # CI: OverflowError: join() result is too long for a Python string
 def test_bytes_exceed_2gb(c_parser_only):
     # see gh-16798
     #

pandas/tests/io/test_sql.py

@@ -1091,7 +1091,7 @@ def test_escaped_table_name(self):
         tm.assert_frame_equal(res, df)
 
 
-@pytest.mark.single
+@pytest.mark.db
 @pytest.mark.skipif(not SQLALCHEMY_INSTALLED, reason="SQLAlchemy not installed")
 class TestSQLApi(SQLAlchemyMixIn, _TestSQLApi):
     """
@@ -1317,12 +1317,12 @@ def setup_method(self, load_iris_data, load_types_data):
         self.pandasSQL = sql.SQLDatabase(self.__engine)
 
 
-@pytest.mark.single
+@pytest.mark.db
 class TestSQLApiConn(_EngineToConnMixin, TestSQLApi):
     pass
 
 
-@pytest.mark.single
+@pytest.mark.db
 class TestSQLiteFallbackApi(SQLiteMixIn, _TestSQLApi):
     """
     Test the public sqlite connection fallback API
@@ -2413,36 +2413,32 @@ def psql_insert_copy(table, conn, keys, data_iter):
         tm.assert_frame_equal(result, expected)
 
 
-@pytest.mark.single
 @pytest.mark.db
 class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy):
     pass
 
 
-@pytest.mark.single
 @pytest.mark.db
 class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn):
     pass
 
 
-@pytest.mark.single
 @pytest.mark.db
 class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy):
     pass
 
 
-@pytest.mark.single
 @pytest.mark.db
 class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn):
     pass
 
 
-@pytest.mark.single
+@pytest.mark.db
 class TestSQLiteAlchemy(_TestSQLiteAlchemy, _TestSQLAlchemy):
     pass
 
 
-@pytest.mark.single
+@pytest.mark.db
 class TestSQLiteAlchemyConn(_TestSQLiteAlchemy, _TestSQLAlchemyConn):
     pass
 
@@ -2451,7 +2447,7 @@ class TestSQLiteAlchemyConn(_TestSQLiteAlchemy, _TestSQLAlchemyConn):
 # -- Test Sqlite / MySQL fallback
 
 
-@pytest.mark.single
+@pytest.mark.db
 class TestSQLiteFallback(SQLiteMixIn, PandasSQLTest):
     """
     Test the fallback mode against an in-memory sqlite database.

pandas/tests/series/methods/test_rank.py

@@ -480,7 +480,6 @@ def test_rank_first_pct(dtype, ser, exp):
 
 
 @pytest.mark.single
-@pytest.mark.high_memory
 def test_pct_max_many_rows():
     # GH 18271
     s = Series(np.arange(2 ** 24 + 1))

pandas/tests/test_algos.py

@@ -1784,7 +1784,6 @@ def test_too_many_ndims(self):
             algos.rank(arr)
 
     @pytest.mark.single
-    @pytest.mark.high_memory
     def test_pct_max_many_rows(self):
         # GH 18271
         values = np.arange(2 ** 24 + 1)

pyproject.toml

@@ -52,7 +52,6 @@ markers = [
   "slow: mark a test as slow",
   "network: mark a test as network",
   "db: tests requiring a database (mysql or postgres)",
-  "high_memory: mark a test as a high-memory only",
   "clipboard: mark a pd.read_clipboard test",
   "arm_slow: mark a test as slow for arm64 architecture",
   "arraymanager: mark a test to run with ArrayManager enabled",