diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1e2556f521225..21cb5bac25be8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,18 +18,19 @@ pr: variables: PYTEST_WORKERS: auto PYTEST_TARGET: pandas + PATTERN: "not slow and not high_memory and not db and not network" + PANDAS_CI: 1 jobs: -# Mac and Linux use the same template - template: ci/azure/posix.yml parameters: name: macOS - vmImage: macOS-10.15 + vmImage: macOS-latest - template: ci/azure/windows.yml parameters: name: Windows - vmImage: windows-2019 + vmImage: windows-latest - job: py38_32bit pool: diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9002f35376613..be76e1366f5ca 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -4,40 +4,22 @@ parameters: jobs: - job: ${{ parameters.name }} + timeoutInMinutes: 90 pool: vmImage: ${{ parameters.vmImage }} strategy: matrix: - py38_macos_1: - ENV_FILE: ci/deps/azure-macos-38.yaml + py38: + ENV_FILE: ci/deps/actions-38.yaml CONDA_PY: "38" - PATTERN: "not slow" - PYTEST_TARGET: "pandas/tests/[a-h]*" - py38_macos_2: - ENV_FILE: ci/deps/azure-macos-38.yaml - CONDA_PY: "38" - PATTERN: "not slow" - PYTEST_TARGET: "pandas/tests/[i-z]*" - py39_macos_1: - ENV_FILE: ci/deps/azure-macos-39.yaml - CONDA_PY: "39" - PATTERN: "not slow" - PYTEST_TARGET: "pandas/tests/[a-h]*" - py39_macos_2: - ENV_FILE: ci/deps/azure-macos-39.yaml + + py39: + ENV_FILE: ci/deps/actions-39.yaml CONDA_PY: "39" - PATTERN: "not slow" - PYTEST_TARGET: "pandas/tests/[i-z]*" - py310_macos_1: - ENV_FILE: ci/deps/azure-macos-310.yaml - CONDA_PY: "310" - PATTERN: "not slow" - PYTEST_TARGET: "pandas/tests/[a-h]*" - py310_macos_2: - ENV_FILE: ci/deps/azure-macos-310.yaml + + py310: + ENV_FILE: ci/deps/actions-310.yaml CONDA_PY: "310" - PATTERN: "not slow" - PYTEST_TARGET: "pandas/tests/[i-z]*" steps: - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 836332251d7b3..3b489d92388da 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -4,51 +4,22 @@ parameters: jobs: - job: ${{ parameters.name }} + timeoutInMinutes: 90 pool: vmImage: ${{ parameters.vmImage }} strategy: matrix: - py38_np18_1: - ENV_FILE: ci/deps/azure-windows-38.yaml + py38: + ENV_FILE: ci/deps/actions-38.yaml CONDA_PY: "38" - PATTERN: "not slow" - PYTEST_WORKERS: 2 # GH-42236 - PYTEST_TARGET: "pandas/tests/[a-h]*" - py38_np18_2: - ENV_FILE: ci/deps/azure-windows-38.yaml - CONDA_PY: "38" - PATTERN: "not slow" - PYTEST_WORKERS: 2 # GH-42236 - PYTEST_TARGET: "pandas/tests/[i-z]*" - - py39_1: - ENV_FILE: ci/deps/azure-windows-39.yaml - CONDA_PY: "39" - PATTERN: "not slow and not high_memory" - PYTEST_WORKERS: 2 # GH-42236 - PYTEST_TARGET: "pandas/tests/[a-h]*" - - py39_2: - ENV_FILE: ci/deps/azure-windows-39.yaml + py39: + ENV_FILE: ci/deps/actions-39.yaml CONDA_PY: "39" - PATTERN: "not slow and not high_memory" - PYTEST_WORKERS: 2 # GH-42236 - PYTEST_TARGET: "pandas/tests/[i-z]*" - - py310_1: - ENV_FILE: ci/deps/azure-windows-310.yaml - CONDA_PY: "310" - PATTERN: "not slow and not high_memory" - PYTEST_WORKERS: 2 # GH-42236 - PYTEST_TARGET: "pandas/tests/[a-h]*" - py310_2: - ENV_FILE: ci/deps/azure-windows-310.yaml + py310: + ENV_FILE: ci/deps/actions-310.yaml CONDA_PY: "310" - PATTERN: "not slow and not high_memory" - PYTEST_WORKERS: 2 # GH-42236 - PYTEST_TARGET: "pandas/tests/[i-z]*" steps: - powershell: | @@ -59,7 +30,7 @@ jobs: displayName: 'Update conda' - bash: | - conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml + conda env create -q --file ci\\deps\\actions-$(CONDA_PY).yaml displayName: 'Create anaconda environment' - bash: | source activate pandas-dev diff --git a/ci/deps/azure-macos-310.yaml b/ci/deps/azure-macos-310.yaml deleted file mode 100644 index 312fac8091db6..0000000000000 --- a/ci/deps/azure-macos-310.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.10 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - html5lib - - jinja2 - - lxml - - matplotlib - - nomkl - - numexpr - - numpy - - openpyxl - - pyarrow - - pyreadstat - - pytables - - python-dateutil==2.8.1 - - pytz - - pyxlsb - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard diff --git a/ci/deps/azure-macos-38.yaml b/ci/deps/azure-macos-38.yaml deleted file mode 100644 index 422aa86c57fc7..0000000000000 --- a/ci/deps/azure-macos-38.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.8 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - html5lib - - jinja2 - - lxml - - matplotlib=3.3.2 - - nomkl - - numexpr - - numpy=1.18.5 - - openpyxl - - pyarrow=1.0.1 - - pyreadstat - - pytables - - python-dateutil==2.8.1 - - pytz - - pyxlsb - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard diff --git a/ci/deps/azure-macos-39.yaml b/ci/deps/azure-macos-39.yaml deleted file mode 100644 index 140d67796452c..0000000000000 --- a/ci/deps/azure-macos-39.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.9 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - html5lib - - jinja2 - - lxml - - matplotlib=3.3.2 - - nomkl - - numexpr - - numpy=1.21.3 - - openpyxl - - pyarrow=4 - - pyreadstat - - pytables - - python-dateutil==2.8.1 - - pytz - - pyxlsb - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard diff --git a/ci/deps/azure-windows-310.yaml b/ci/deps/azure-windows-310.yaml deleted file mode 100644 index 8e6f4deef6057..0000000000000 --- a/ci/deps/azure-windows-310.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: pandas-dev -channels: - - conda-forge - - defaults -dependencies: - - python=3.10 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - fsspec>=0.8.0 - - gcsfs - - html5lib - - jinja2 - - lxml - - matplotlib - # TODO: uncomment after numba supports py310 - #- numba - - numexpr - - numpy - - openpyxl - - pyarrow - - pytables - - python-dateutil - - pytz - - s3fs>=0.4.2 - - scipy - - sqlalchemy - - xlrd - - xlsxwriter - - xlwt - - pyreadstat - - pyxlsb - - zstandard diff --git a/ci/deps/azure-windows-38.yaml b/ci/deps/azure-windows-38.yaml deleted file mode 100644 index eb533524147d9..0000000000000 --- a/ci/deps/azure-windows-38.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: pandas-dev -channels: - - conda-forge - - defaults -dependencies: - - python=3.8 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - blosc - - bottleneck - - fastparquet>=0.4.0 - - fsspec>=0.8.0 - - matplotlib=3.3.2 - - numba - - numexpr - - numpy=1.18 - - openpyxl - - jinja2 - - pyarrow=2 - - pytables - - python-dateutil - - pytz - - s3fs>=0.4.0 - - scipy - - xlrd - - xlsxwriter - - xlwt - - zstandard diff --git a/ci/deps/azure-windows-39.yaml b/ci/deps/azure-windows-39.yaml deleted file mode 100644 index 6f820b1c2aedb..0000000000000 --- a/ci/deps/azure-windows-39.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: pandas-dev -channels: - - conda-forge - - defaults -dependencies: - - python=3.9 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - fsspec>=0.8.0 - - gcsfs - - html5lib - - jinja2 - - lxml - - matplotlib - - numba - - numexpr - - numpy - - openpyxl - - pyarrow=6 - - pytables - - python-dateutil - - pytz - - s3fs>=0.4.2 - - scipy - - sqlalchemy - - xlrd - - xlsxwriter - - xlwt - - pyreadstat - - pyxlsb - - zstandard diff --git a/ci/run_tests.sh b/ci/run_tests.sh index 020a3ed7f265c..786bb0c6b6150 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -24,18 +24,14 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" - -if [[ $(uname) != "Linux" && $(uname) != "Darwin" ]]; then - PYTEST_CMD="$PYTEST_CMD --ignore=pandas/tests/plotting/" -fi +PYTEST_CMD="${XVFB}pytest -v -r fEs -m \"$PATTERN\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" echo $PYTEST_CMD sh -c "$PYTEST_CMD" if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array - PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" + PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -v -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" echo $PYTEST_AM_CMD sh -c "$PYTEST_AM_CMD" diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index adf4f32837acf..844eb5521208a 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -236,10 +236,13 @@ def test_write_missing_parent_directory(self, method, module, error_class, fn_ex path = os.path.join(HERE, "data", "missing_folder", "does_not_exist." + fn_ext) - with pytest.raises( - error_class, - match=r"Cannot save file into a non-existent directory: .*missing_folder", - ): + msg = "|".join( + [ + r"\[Errno 2\] No such", + "Cannot save file into a non-existent directory: .*missing_folder", + ] + ) + with pytest.raises(error_class, match=msg): method(dummy_frame, path) @pytest.mark.parametrize( diff --git a/pandas/tests/io/test_fsspec.py b/pandas/tests/io/test_fsspec.py index f1040c0bd30f2..c3f2a0c176b78 100644 --- a/pandas/tests/io/test_fsspec.py +++ b/pandas/tests/io/test_fsspec.py @@ -3,6 +3,7 @@ import numpy as np import pytest +from pandas.compat import is_platform_mac from pandas.compat._optional import VERSIONS from pandas import ( @@ -161,6 +162,14 @@ def test_to_parquet_new_file(monkeypatch, cleared_fs): @td.skip_if_no("pyarrow", min_version="2") +@pytest.mark.xfail( + is_platform_mac(), + reason="Incorrect build can lead to " + "dlopen(.../pyarrow/_parquet.cpython-39-darwin.so, 2): " + "Library not loaded: @rpath/libssl.1.1.dylib", + raises=ImportError, + strict=False, +) def test_arrowparquet_options(fsspectest): """Regression test for writing to a not-yet-existent GCS Parquet file.""" df = DataFrame({"a": [0]}) diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py index b57923093e3e8..cbf390947d0ec 100644 --- a/pandas/tests/io/test_parquet.py +++ b/pandas/tests/io/test_parquet.py @@ -3,6 +3,7 @@ from io import BytesIO import os import pathlib +import sys from warnings import ( catch_warnings, filterwarnings, @@ -13,7 +14,10 @@ from pandas._config import get_option -from pandas.compat import is_platform_windows +from pandas.compat import ( + is_platform_mac, + is_platform_windows, +) from pandas.compat.pyarrow import ( pa_version_under2p0, pa_version_under5p0, @@ -54,9 +58,17 @@ _HAVE_FASTPARQUET = False -pytestmark = pytest.mark.filterwarnings( - "ignore:RangeIndex.* is deprecated:DeprecationWarning" -) +pytestmark = [ + pytest.mark.filterwarnings("ignore:RangeIndex.* is deprecated:DeprecationWarning"), + pytest.mark.xfail( + is_platform_mac(), + reason="Incorrect build can lead to " + "dlopen(.../pyarrow/_parquet.cpython-39-darwin.so, 2): " + "Library not loaded: @rpath/libssl.1.1.dylib", + raises=ImportError, + strict=False, + ), +] # TODO(ArrayManager) fastparquet relies on BlockManager internals @@ -171,6 +183,7 @@ def check_round_trip( check_names=True, check_like=False, check_dtype=True, + check_index_type="equiv", repeat=2, ): """Verify parquet serializer and deserializer produce the same results. @@ -217,6 +230,7 @@ def compare(repeat): check_names=check_names, check_like=check_like, check_dtype=check_dtype, + check_index_type=check_index_type, ) if path is None: @@ -836,6 +850,11 @@ def test_s3_roundtrip_for_dir( ) @td.skip_if_no("pyarrow") + @pytest.mark.xfail( + is_platform_mac() and sys.version_info[:2] == (3, 9), + raises=TypeError, + reason="expected str, bytes or os.PathLike object, not BytesIO", + ) def test_read_file_like_obj_support(self, df_compat): buffer = BytesIO() df_compat.to_parquet(buffer) @@ -947,6 +966,7 @@ def test_timezone_aware_index(self, request, pa, timezone_aware_date_list): if ( not pa_version_under2p0 and timezone_aware_date_list.tzinfo != datetime.timezone.utc + and not is_platform_windows() ): request.node.add_marker( pytest.mark.xfail( @@ -958,7 +978,7 @@ def test_timezone_aware_index(self, request, pa, timezone_aware_date_list): df = pd.DataFrame(index=idx, data={"index_as_col": idx}) # see gh-36004 - # compare time(zone) values only, skip their class: + # compare time(zone) values only, skip their class in the values and index: # pyarrow always creates fixed offset timezones using pytz.FixedOffset() # even if it was datetime.timezone() originally # @@ -966,7 +986,7 @@ def test_timezone_aware_index(self, request, pa, timezone_aware_date_list): # they both implement datetime.tzinfo # they both wrap datetime.timedelta() # this use-case sets the resolution to 1 minute - check_round_trip(df, pa, check_dtype=False) + check_round_trip(df, pa, check_dtype=False, check_index_type=False) @td.skip_if_no("pyarrow", min_version="1.0.0") def test_filter_row_groups(self, pa):