CI/TST: Misc test cleanups (#53931)

* TST: Misc testing cleanups

* more cleans

* pytest updates

* trigger ci

* Remove tmp_file removal

* Revert gcs testing

* type ignore

Author: mroeschke

.github/workflows/unit-tests.yml

@@ -81,7 +81,7 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
-      LANG: ${{ matrix.lang || '' }}
+      LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
@@ -122,7 +122,7 @@ jobs:
           - 5432:5432
 
       moto:
-        image: motoserver/moto:4.1.4
+        image: motoserver/moto:4.1.12
         env:
           AWS_ACCESS_KEY_ID: foobar_key
           AWS_SECRET_ACCESS_KEY: foobar_secret

ci/code_checks.sh

@@ -21,23 +21,6 @@ BASE_DIR="$(dirname $0)/.."
 RET=0
 CHECK=$1
 
-function invgrep {
-    # grep with inverse exist status and formatting for azure-pipelines
-    #
-    # This function works exactly as grep, but with opposite exit status:
-    # - 0 (success) when no patterns are found
-    # - 1 (fail) when the patterns are found
-    #
-    # This is useful for the CI, as we want to fail if one of the patterns
-    # that we want to avoid is found by grep.
-    grep -n "$@" | sed "s/^/$INVGREP_PREPEND/" | sed "s/$/$INVGREP_APPEND/" ; EXIT_STATUS=${PIPESTATUS[0]}
-    return $((! $EXIT_STATUS))
-}
-
-if [[ "$GITHUB_ACTIONS" == "true" ]]; then
-    INVGREP_PREPEND="##[error]"
-fi
-
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 

ci/run_tests.sh

@@ -8,19 +8,9 @@ export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 429496
 # May help reproduce flaky CI builds if set in subsequent runs
 echo PYTHONHASHSEED=$PYTHONHASHSEED
 
-if [[ "not network" == *"$PATTERN"* ]]; then
-    export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
-fi
-
 COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
-# If no X server is found, we use xvfb to emulate it
-if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
-    export DISPLAY=":0"
-    XVFB="xvfb-run "
-fi
-
-PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 ${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""

pandas/_testing/_io.py

@@ -3,6 +3,7 @@
 import bz2
 import gzip
 import io
+import pathlib
 import tarfile
 from typing import (
     TYPE_CHECKING,
@@ -77,14 +78,12 @@ def round_trip_pathlib(writer, reader, path: str | None = None):
     pandas object
         The original object that was serialized and then re-read.
     """
-    import pytest
-
-    Path = pytest.importorskip("pathlib").Path
+    Path = pathlib.Path
     if path is None:
         path = "___pathlib___"
     with ensure_clean(path) as path:
-        writer(Path(path))
-        obj = reader(Path(path))
+        writer(Path(path))  # type: ignore[arg-type]
+        obj = reader(Path(path))  # type: ignore[arg-type]
     return obj
 
 

pandas/tests/groupby/test_timegrouper.py

@@ -907,6 +907,7 @@ def test_groupby_apply_timegrouper_with_nat_apply_squeeze(
         tm.assert_frame_equal(res, expected)
 
     @td.skip_if_no("numba")
+    @pytest.mark.single_cpu
     def test_groupby_agg_numba_timegrouper_with_nat(
         self, groupby_with_truncated_bingrouper
     ):

pandas/tests/io/parser/test_network.py

@@ -74,11 +74,6 @@ def tips_df(datapath):
 
 @pytest.mark.single_cpu
 @pytest.mark.usefixtures("s3_resource")
-@pytest.mark.xfail(
-    reason="CI race condition GH 45433, GH 44584",
-    raises=FileNotFoundError,
-    strict=False,
-)
 @td.skip_if_not_us_locale()
 class TestS3:
     @td.skip_if_no("s3fs")

pyproject.toml

@@ -453,7 +453,7 @@ disable = [
 
 [tool.pytest.ini_options]
 # sync minversion with pyproject.toml & install.rst
-minversion = "7.0"
+minversion = "7.3.2"
 addopts = "--strict-data-files --strict-markers --strict-config --capture=no --durations=30 --junitxml=test-data.xml"
 empty_parameter_set_mark = "fail_at_collect"
 xfail_strict = true