From 2e829efc35fca822e2aa553388fe2914c4654013 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 22 Feb 2022 11:13:13 -0800 Subject: [PATCH 1/5] CI: Align MacOS dependencies with other builds --- azure-pipelines.yml | 1 - ci/azure/posix.yml | 6 +++--- ci/deps/azure-macos-310.yaml | 36 ------------------------------------ ci/deps/azure-macos-38.yaml | 36 ------------------------------------ ci/deps/azure-macos-39.yaml | 36 ------------------------------------ 5 files changed, 3 insertions(+), 112 deletions(-) delete mode 100644 ci/deps/azure-macos-310.yaml delete mode 100644 ci/deps/azure-macos-38.yaml delete mode 100644 ci/deps/azure-macos-39.yaml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 747be0654ac70..dc0714871503c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,6 @@ variables: PANDAS_CI: 1 jobs: -# Mac and Linux use the same template - template: ci/azure/posix.yml parameters: name: macOS diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 6d33cd7e5f5d9..7617baae14966 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -10,15 +10,15 @@ jobs: strategy: matrix: py38: - ENV_FILE: ci/deps/azure-macos-38.yaml + ENV_FILE: ci/deps/actions-38.yaml CONDA_PY: "38" py39: - ENV_FILE: ci/deps/azure-macos-39.yaml + ENV_FILE: ci/deps/actions-38.yaml CONDA_PY: "39" py310: - ENV_FILE: ci/deps/azure-macos-310.yaml + ENV_FILE: ci/deps/actions-38.yaml CONDA_PY: "310" steps: diff --git a/ci/deps/azure-macos-310.yaml b/ci/deps/azure-macos-310.yaml deleted file mode 100644 index 312fac8091db6..0000000000000 --- a/ci/deps/azure-macos-310.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.10 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - html5lib - - jinja2 - - lxml - - matplotlib - - nomkl - - numexpr - - numpy - - openpyxl - - pyarrow - - pyreadstat - - pytables - - python-dateutil==2.8.1 - - pytz - - pyxlsb - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard diff --git a/ci/deps/azure-macos-38.yaml b/ci/deps/azure-macos-38.yaml deleted file mode 100644 index 422aa86c57fc7..0000000000000 --- a/ci/deps/azure-macos-38.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.8 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - html5lib - - jinja2 - - lxml - - matplotlib=3.3.2 - - nomkl - - numexpr - - numpy=1.18.5 - - openpyxl - - pyarrow=1.0.1 - - pyreadstat - - pytables - - python-dateutil==2.8.1 - - pytz - - pyxlsb - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard diff --git a/ci/deps/azure-macos-39.yaml b/ci/deps/azure-macos-39.yaml deleted file mode 100644 index 140d67796452c..0000000000000 --- a/ci/deps/azure-macos-39.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: pandas-dev -channels: - - defaults - - conda-forge -dependencies: - - python=3.9 - - # tools - - cython>=0.29.24 - - pytest>=6.0 - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 - - pytest-azurepipelines - - # pandas dependencies - - beautifulsoup4 - - bottleneck - - html5lib - - jinja2 - - lxml - - matplotlib=3.3.2 - - nomkl - - numexpr - - numpy=1.21.3 - - openpyxl - - pyarrow=4 - - pyreadstat - - pytables - - python-dateutil==2.8.1 - - pytz - - pyxlsb - - xarray - - xlrd - - xlsxwriter - - xlwt - - zstandard From 8f3c0d45623c1c6490713d985e8e44a08fb43f48 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 22 Feb 2022 11:15:54 -0800 Subject: [PATCH 2/5] fix file names --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 7617baae14966..be76e1366f5ca 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -14,11 +14,11 @@ jobs: CONDA_PY: "38" py39: - ENV_FILE: ci/deps/actions-38.yaml + ENV_FILE: ci/deps/actions-39.yaml CONDA_PY: "39" py310: - ENV_FILE: ci/deps/actions-38.yaml + ENV_FILE: ci/deps/actions-310.yaml CONDA_PY: "310" steps: From 4d9bd709025508446a64e9bbfc63a7f861a70888 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 22 Feb 2022 18:14:48 -0800 Subject: [PATCH 3/5] Try installing libthrift --- ci/azure/posix.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index be76e1366f5ca..5225af2aa5148 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,6 +28,13 @@ jobs: - script: ci/setup_env.sh displayName: 'Setup environment and build pandas' + # From pyarrow + # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib + # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib + # Reason: image not found + - script: conda install -n pandas-dev -c conda-forge --no-update-deps libthrift + displayName: 'Re-install libthrift' + - script: | source activate pandas-dev ci/run_tests.sh From 9a6be6e8af0b609e9e84cba3442154a444bc5421 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Tue, 22 Feb 2022 20:15:08 -0800 Subject: [PATCH 4/5] Do it in setup_env.sh --- ci/azure/posix.yml | 7 ------- ci/setup_env.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 5225af2aa5148..be76e1366f5ca 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,13 +28,6 @@ jobs: - script: ci/setup_env.sh displayName: 'Setup environment and build pandas' - # From pyarrow - # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib - # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib - # Reason: image not found - - script: conda install -n pandas-dev -c conda-forge --no-update-deps libthrift - displayName: 'Re-install libthrift' - - script: | source activate pandas-dev ci/run_tests.sh diff --git a/ci/setup_env.sh b/ci/setup_env.sh index d51ff98b241a6..73f9a656c379d 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -66,6 +66,14 @@ echo echo "mamba env create -q --file=${ENV_FILE}" time mamba env create -q --file="${ENV_FILE}" +# From pyarrow on MacOS +# ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib +# Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib +# Reason: image not found +if [[ "$(uname)" == 'Darwin' ]]; then + echo "Update libthrift for pyarrow on MacOS" + conda install -n pandas-dev -c conda-forge --no-update-deps libthrift=0.15.0 +fi if [[ "$BITS32" == "yes" ]]; then # activate 32-bit compiler From 5a3ac81ce906fa1661b0c2d2939784b6ec7eccd6 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Wed, 23 Feb 2022 11:10:20 -0800 Subject: [PATCH 5/5] Upgrade pyarrow --- ci/setup_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/setup_env.sh b/ci/setup_env.sh index 73f9a656c379d..bb9656063db3f 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -71,8 +71,8 @@ time mamba env create -q --file="${ENV_FILE}" # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib # Reason: image not found if [[ "$(uname)" == 'Darwin' ]]; then - echo "Update libthrift for pyarrow on MacOS" - conda install -n pandas-dev -c conda-forge --no-update-deps libthrift=0.15.0 + echo "Update pyarrow for pyarrow on MacOS" + conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=6 fi if [[ "$BITS32" == "yes" ]]; then