From 0e4250e5119c4856f116ed6a0a3464b4c9c5d3a9 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 16 Jul 2021 21:51:21 -0500 Subject: [PATCH 01/10] CI: clean Azure Windows --- ci/azure/windows.yml | 61 ++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 05ba7c57ad6c4..463c0f1668d64 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -9,47 +9,54 @@ jobs: strategy: matrix: py38_np18: - ENV_FILE: ci/deps/azure-windows-38.yaml - CONDA_PY: "38" + ENV_FILE: ci\\deps\\azure-windows-38.yaml PATTERN: "not slow and not network" - PYTEST_WORKERS: 2 # GH-42236 + # PYTEST_WORKERS: 2 # GH-42236 py39: - ENV_FILE: ci/deps/azure-windows-39.yaml - CONDA_PY: "39" + ENV_FILE: ci\\deps\\azure-windows-39.yaml PATTERN: "not slow and not network and not high_memory" - PYTEST_WORKERS: 2 # GH-42236 + # PYTEST_WORKERS: 2 # GH-42236 steps: - - powershell: | + - script: | Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" - displayName: 'Add conda to PATH' - - script: conda update -q -n base conda - displayName: 'Update conda' - - - bash: | - conda env create -q --file ci\\deps\\azure-windows-$(CONDA_PY).yaml - displayName: 'Create anaconda environment' - - bash: | - source activate pandas-dev + displayName: Add conda to PATH + + - script: | + conda update --quiet --yes conda + displayName: Update conda + + - script: | + conda env create -q --file $(ENV_FILE) + displayName: Create Anaconda environment + + - script: | + call activate pandas-dev conda list python setup.py build_ext -q -j 4 - python -m pip install --no-build-isolation -e . - displayName: 'Build' - - bash: | - source activate pandas-dev + python -m pip install -e . --no-build-isolation --no-use-pep517 + displayName: Build pandas + + - script: | + call activate pandas-dev ci/run_tests.sh - displayName: 'Test' + displayName: Test + + - script: | + call activate pandas-dev + python -c "import pandas; pandas.show_versions();" + displayName: Build versions + - task: PublishTestResults@2 condition: succeededOrFailed() inputs: failTaskOnFailedTests: true testResultsFiles: 'test-data.xml' - testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} - displayName: 'Publish test results' + testRunTitle: ${{ format('{0}-$(ENV_FILE)', parameters.name) }} + displayName: Publish test results - - bash: | - source activate pandas-dev + - script: | + call activate pandas-dev python ci/print_skipped.py - displayName: 'Print skipped tests' + displayName: Print skipped tests From a01f4c815e2e60a4ca4cf552cc5873b6f55d6780 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 16 Jul 2021 21:58:22 -0500 Subject: [PATCH 02/10] switch back to powershell --- ci/azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 463c0f1668d64..591bcbbcf4389 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -19,7 +19,7 @@ jobs: # PYTEST_WORKERS: 2 # GH-42236 steps: - - script: | + - powershell: | Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" displayName: Add conda to PATH From 5199d9add8bd480f209a15caa7e070cfbae4ceab Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 16 Jul 2021 23:05:33 -0500 Subject: [PATCH 03/10] fix test task --- ci/azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 591bcbbcf4389..65f37011c5bf6 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -40,7 +40,7 @@ jobs: - script: | call activate pandas-dev - ci/run_tests.sh + sh ci/run_tests.sh displayName: Test - script: | From f8206f3e29d487f6d5eab4cc383260e5acd3a51e Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 16 Jul 2021 23:32:16 -0500 Subject: [PATCH 04/10] debug --- ci/azure/windows.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 65f37011c5bf6..f21d2c9197e59 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -33,7 +33,6 @@ jobs: - script: | call activate pandas-dev - conda list python setup.py build_ext -q -j 4 python -m pip install -e . --no-build-isolation --no-use-pep517 displayName: Build pandas From 192ebf97830b421014ff36e4e2f009a1fdeacfdd Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 20 Jul 2021 21:25:43 -0500 Subject: [PATCH 05/10] clean --- ci/azure/windows.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index f21d2c9197e59..211481d8bd2a4 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -11,12 +11,10 @@ jobs: py38_np18: ENV_FILE: ci\\deps\\azure-windows-38.yaml PATTERN: "not slow and not network" - # PYTEST_WORKERS: 2 # GH-42236 py39: ENV_FILE: ci\\deps\\azure-windows-39.yaml PATTERN: "not slow and not network and not high_memory" - # PYTEST_WORKERS: 2 # GH-42236 steps: - powershell: | @@ -33,7 +31,7 @@ jobs: - script: | call activate pandas-dev - python setup.py build_ext -q -j 4 + python setup.py build_ext -q -j 2 python -m pip install -e . --no-build-isolation --no-use-pep517 displayName: Build pandas From bbd405498fad00dd05cda8e1c668ef33d1f5620a Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 20 Jul 2021 21:28:25 -0500 Subject: [PATCH 06/10] add cache --- azure-pipelines.yml | 1 + ci/azure/windows.yml | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e16fd36ac8f98..d6f1edf40c5e5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,7 @@ pr: variables: PYTEST_WORKERS: auto + CONDA_CACHE_DIR: $(Pipeline.Workspace)/.condarc jobs: # Mac and Linux use the same template diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 211481d8bd2a4..0246911c63550 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -25,9 +25,20 @@ jobs: conda update --quiet --yes conda displayName: Update conda + - task: Cache@2 + displayName: Use cached Anaconda environment + inputs: + key: 'conda | "$(Agent.OS)" | environment.yml' + restoreKeys: | + python | "$(Agent.OS)" + python + path: $(CONDA_CACHE_DIR) + cacheHitVar: CONDA_CACHE_RESTORED + - script: | conda env create -q --file $(ENV_FILE) displayName: Create Anaconda environment + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - script: | call activate pandas-dev From 5d079c88922da5cb62a92916fc38407bcddd0957 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 20 Jul 2021 21:31:37 -0500 Subject: [PATCH 07/10] fix keys --- ci/azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 0246911c63550..504f15fe95e9f 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -28,7 +28,7 @@ jobs: - task: Cache@2 displayName: Use cached Anaconda environment inputs: - key: 'conda | "$(Agent.OS)" | environment.yml' + key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' restoreKeys: | python | "$(Agent.OS)" python From 43df568638e0a7288d2acb783d70ebb773bd720b Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 20 Jul 2021 22:30:07 -0500 Subject: [PATCH 08/10] test cache --- ci/azure/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 504f15fe95e9f..fd256efe3e3ef 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -29,7 +29,7 @@ jobs: displayName: Use cached Anaconda environment inputs: key: 'conda | "$(Agent.OS)" | "$(ENV_FILE)"' - restoreKeys: | + restoreKeys: | python | "$(Agent.OS)" python path: $(CONDA_CACHE_DIR) From 26ae6ffe62a023df354f159762af4f645fdd82fc Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 21 Jul 2021 10:09:15 -0500 Subject: [PATCH 09/10] fix path --- azure-pipelines.yml | 1 - ci/azure/windows.yml | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d6f1edf40c5e5..e16fd36ac8f98 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,7 +17,6 @@ pr: variables: PYTEST_WORKERS: auto - CONDA_CACHE_DIR: $(Pipeline.Workspace)/.condarc jobs: # Mac and Linux use the same template diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index fd256efe3e3ef..2ee2ab505d297 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -2,6 +2,9 @@ parameters: name: '' vmImage: '' +variables: + CONDA_CACHE_DIR: $(Pipeline.Workspace)\\.condarc + jobs: - job: ${{ parameters.name }} pool: From 2f16748138ad03ecc5de372f25f6ebdb4362eabc Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 21 Jul 2021 11:18:30 -0500 Subject: [PATCH 10/10] fix path --- azure-pipelines.yml | 1 + ci/azure/windows.yml | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e16fd36ac8f98..6aa78128f73fc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,7 @@ pr: variables: PYTEST_WORKERS: auto + CONDA_CACHE_DIR: $(Pipeline.Workspace)\\.condarc jobs: # Mac and Linux use the same template diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index 2ee2ab505d297..fd256efe3e3ef 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -2,9 +2,6 @@ parameters: name: '' vmImage: '' -variables: - CONDA_CACHE_DIR: $(Pipeline.Workspace)\\.condarc - jobs: - job: ${{ parameters.name }} pool: