From d1d10248bc6baea1e5b309cf9c516ab6a9f0dc22 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Thu, 24 Feb 2022 14:48:02 +0100 Subject: [PATCH 01/75] Add ccache to GHA builds - Refactor duplicated code from GHA workflows to the 'build-pandas' and 'setup' actions. - Use Mamba over Conda everywhere. - Add sccache using 'hendrikmuhs/ccache-action'. - General cleanup of GHA workflows. --- .github/actions/build-pandas/action.yml | 52 ++++++++++++++++++ .github/actions/build_pandas/action.yml | 17 ------ .github/actions/setup-sccache/action.yml | 18 +++++++ .github/actions/setup/action.yml | 64 ++++++++++++++++++++--- .github/workflows/asv-bot.yml | 21 ++------ .github/workflows/code-checks.yml | 46 ++++------------ .github/workflows/docbuild-and-upload.yml | 18 ++++--- .github/workflows/posix.yml | 46 +++------------- .github/workflows/python-dev.yml | 25 +++------ .github/workflows/sdist.yml | 21 ++++---- setup.py | 43 ++++++++++++++- 11 files changed, 221 insertions(+), 150 deletions(-) create mode 100644 .github/actions/build-pandas/action.yml delete mode 100644 .github/actions/build_pandas/action.yml create mode 100644 .github/actions/setup-sccache/action.yml diff --git a/.github/actions/build-pandas/action.yml b/.github/actions/build-pandas/action.yml new file mode 100644 index 0000000000000..e3e393c64544e --- /dev/null +++ b/.github/actions/build-pandas/action.yml @@ -0,0 +1,52 @@ +name: Build pandas +description: Rebuilds the C extensions and installs pandas +inputs: + use-login-shell: + description: "Use 'bash -l' as shell (required for Conda envs)" + default: true +runs: + using: composite + steps: + # Create a shell wrapper to be able to call "bash" or "bash -l" depending + # on the "use-login-shell" arguments. + # We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments. + - name: Set shell + shell: bash + run: | + if [ ${{ inputs.use-login-shell }} = true ]; then + args="-l" + fi + echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell + cat /tmp/_build_pandas_shell + + - name: Environment Detail + shell: bash /tmp/_build_pandas_shell {0} + run: | + if which conda; then + conda info + conda list + fi + if which pip; then + pip list + fi + python --version + + - name: Get Python version + id: get-python-version + shell: bash /tmp/_build_pandas_shell {0} + run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')" + + - name: Set up sccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ steps.get-python-version.outputs.version }} + + - name: Build Pandas + shell: bash /tmp/_build_pandas_shell {0} + run: | + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -vv -j 2 + python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index + + - name: Build Version + shell: bash /tmp/_build_pandas_shell {0} + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml deleted file mode 100644 index 2e4bfea165316..0000000000000 --- a/.github/actions/build_pandas/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Build pandas -description: Rebuilds the C extensions and installs pandas -runs: - using: composite - steps: - - - name: Environment Detail - run: | - conda info - conda list - shell: bash -l {0} - - - name: Build Pandas - run: | - python setup.py build_ext -j 2 - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index - shell: bash -l {0} diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml new file mode 100644 index 0000000000000..c954d185f72fa --- /dev/null +++ b/.github/actions/setup-sccache/action.yml @@ -0,0 +1,18 @@ +name: Setup sccache +inputs: + extra-cache-key: + required: false + default: '' +runs: + using: composite + steps: + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash + + - name: Setup sccache + uses: jonashaag/ccache-action@sccache-2 + with: + sccache: true + key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9ef00e7a85a6f..dd0a618a4dc71 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,12 +1,64 @@ name: Set up pandas description: Runs all the setup steps required to have a built pandas ready to use +inputs: + environment-file: + default: environment.yml + pyarrow-version: + required: false + is-pypy: + default: false + activate-environment: + default: pandas-dev + python-version: + required: false runs: using: composite steps: - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - shell: bash -l {0} + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash - - name: Setup environment and build pandas - run: ci/setup_env.sh - shell: bash -l {0} + - name: Cache Conda packages + uses: actions/cache@v2 + with: + path: ~/conda_pkgs_dir + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }} + + - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} + run: | + grep -q '\- pyarrow' ${{ inputs.environment-file }} + sed -i "s/- pyarrow/- pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} + cat ${{ inputs.environment-file }} + shell: bash + if: ${{ inputs.pyarrow-version }} + + - name: Setup Mambaforge and install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }}) + uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "0.21.2" + use-mamba: true + channels: conda-forge + activate-environment: ${{ inputs.activate-environment }} + channel-priority: strict + environment-file: ${{ inputs.environment-file }} + python-version: ${{ inputs.python-version }} + use-only-tar-bz2: true + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Pin setuptools (GH#44980) + run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0' + shell: bash + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Setup PyPy + uses: actions/setup-python@v2 + with: + python-version: "pypy-3.8" + if: ${{ inputs.is-pypy == 'true' }} + + - name: Setup PyPy dependencies + # TODO: re-enable cov, its slowing the tests down though + run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 + shell: bash + if: ${{ inputs.is-pypy == 'true' }} diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index f3946aeb84a63..36168bee50687 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -6,8 +6,7 @@ on: - created env: - ENV_FILE: environment.yml - COMMENT: ${{github.event.comment.body}} + COMMENT: ${{ github.event.comment.body }} jobs: autotune: @@ -33,20 +32,10 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - # Although asv sets up its own env, deps are still needed - # during discovery process - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + # Although asv sets up its own env, deps are still needed + # during discovery process + - name: Set up Conda + uses: ./.github/actions/setup - name: Run benchmarks id: bench diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 58bef05940af1..f097c4d53f99e 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -11,7 +11,6 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: @@ -52,20 +51,13 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + - name: Set up Conda + uses: ./.github/actions/setup - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Build Pandas + uses: ./.github/actions/build-pandas + id: build + continue-on-error: true - name: Install node.js (for pyright) uses: actions/setup-node@v2 @@ -76,10 +68,6 @@ jobs: # note: keep version in sync with .pre-commit-config.yaml run: npm install -g pyright@1.1.212 - - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas - - name: Run checks on imported code run: ci/code_checks.sh code if: ${{ steps.build.outcome == 'success' }} @@ -94,11 +82,9 @@ jobs: - name: Run typing validation run: ci/code_checks.sh typing - if: ${{ steps.build.outcome == 'success' }} - name: Run docstring validation script tests run: pytest scripts - if: ${{ steps.build.outcome == 'success' }} asv-benchmarks: name: ASV Benchmarks @@ -118,24 +104,11 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Set up Conda + uses: ./.github/actions/setup - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas + uses: ./.github/actions/build-pandas - name: Run ASV benchmarks run: | @@ -148,7 +121,6 @@ jobs: if grep "failed" benchmarks.log > /dev/null ; then exit 1 fi - if: ${{ steps.build.outcome == 'success' }} - name: Publish benchmarks artifact uses: actions/upload-artifact@v2 diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 4cce75779d750..1d8d171078c07 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -11,13 +11,15 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: web_and_docs: name: Doc Build and Upload runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -30,17 +32,17 @@ jobs: with: fetch-depth: 0 - - name: Set up pandas + - name: Set up Conda uses: ./.github/actions/setup + - name: Build pandas + uses: ./.github/actions/build-pandas + - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build + run: python web/pandas_web.py web/pandas --target-path=web/build + - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors + run: doc/make.py --warnings-are-errors - name: Install ssh key run: | diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 4380e8dfa2e57..158b7d578bc0c 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -14,6 +14,7 @@ on: env: PANDAS_CI: 1 + PYTEST_TARGET: pandas jobs: pytest: @@ -125,57 +126,24 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} - - name: Extra installs # xsel for clipboard tests run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + uses: ./.github/actions/setup with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: flexible environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support + pyarrow-version: ${{ matrix.pyarrow_version }} + is-pypy: ${{ env.IS_PYPY }} - - name: Upgrade Arrow version - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} - if: ${{ matrix.pyarrow_version }} - - - name: Setup PyPy - uses: actions/setup-python@v2 - with: - python-version: "pypy-3.8" - if: ${{ env.IS_PYPY == 'true' }} - - - name: Setup PyPy dependencies - shell: bash - run: | - # TODO: re-enable cov, its slowing the tests down though - pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 - if: ${{ env.IS_PYPY == 'true' }} - - - name: Build Pandas - uses: ./.github/actions/build_pandas + - name: Build pandas + uses: ./.github/actions/build-pandas - name: Test run: ci/run_tests.sh # TODO: Don't continue on error for PyPy continue-on-error: ${{ env.IS_PYPY == 'true' }} - if: always() - - - name: Build Version - run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - name: Publish test results uses: actions/upload-artifact@v2 diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c287827206336..3ec5aaf9372b9 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,11 +21,11 @@ on: - "doc/**" env: - PYTEST_WORKERS: "auto" PANDAS_CI: 1 + PYTEST_WORKERS: "auto" + PYTEST_TARGET: pandas PATTERN: "not slow and not network and not clipboard and not single_cpu" COVERAGE: true - PYTEST_TARGET: pandas jobs: build: @@ -56,27 +56,19 @@ jobs: # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install dependencies - shell: bash run: | python -m pip install --upgrade pip "setuptools<60.0.0" wheel pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy pip install git+https://github.com/nedbat/coveragepy.git pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" + - name: Build pandas + uses: ./.github/actions/build-pandas + with: + use-login-shell: false - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh + run: bash ci/run_tests.sh - name: Publish test results uses: actions/upload-artifact@v2 @@ -86,8 +78,7 @@ jobs: if: failure() - name: Report Coverage - run: | - coverage report -m + run: coverage report -m - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 431710a49a7dd..71315947bd4f0 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -60,18 +60,24 @@ jobs: name: ${{matrix.python-version}}-sdist.gz path: dist/*.gz - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda + uses: ./.github/actions/setup with: + environment-file: "" activate-environment: pandas-sdist - channels: conda-forge - python-version: '${{ matrix.python-version }}' + python-version: ${{ matrix.python-version }} + + - name: Set up ccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ matrix.python-version }} # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install pandas from sdist run: | python -m pip install --upgrade "setuptools<60.0.0" pip list - python -m pip install dist/*.gz + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python -m pip install -vv dist/*.gz - name: Force oldest supported NumPy run: | @@ -84,8 +90,5 @@ jobs: pip install numpy==1.21.2 ;; esac - - name: Import pandas - run: | - cd .. - conda list - python -c "import pandas; pandas.show_versions();" + - name: Build Version + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/setup.py b/setup.py index 62704dc4423c8..cd8a5a49322b6 100755 --- a/setup.py +++ b/setup.py @@ -74,7 +74,48 @@ def is_platform_mac(): _pxi_dep[module] = pxi_files -class build_ext(_build_ext): +class CompilerLauncherMixin: + """Add "compiler launchers" to distutils. + + We use this to be able to run the Pandas build using "ccache". + + A compiler launcher is a program that is invoked instead of invoking the + compiler directly. It is passed the full compiler invocation command line. + + A similar feature exists in CMake, see + https://cmake.org/cmake/help/latest/prop_tgt/LANG_COMPILER_LAUNCHER.html. + """ + + __is_set_up = False + + def build_extensions(self): + # Integrate into "build_ext" + self.__setup() + super().build_extensions() + + def build_libraries(self): + # Integrate into "build_clib" + self.__setup() + super().build_extensions() + + def __setup(self): + if self.__is_set_up: + return + self.__is_set_up = True + compiler_launcher = os.getenv("DISTUTILS_C_COMPILER_LAUNCHER") + if compiler_launcher: + + def spawn_with_compiler_launcher(cmd): + exclude_programs = ("link.exe",) + if not cmd[0].endswith(exclude_programs): + cmd = [compiler_launcher] + cmd + return original_spawn(cmd) + + original_spawn = self.compiler.spawn + self.compiler.spawn = spawn_with_compiler_launcher + + +class build_ext(CompilerLauncherMixin, _build_ext): @classmethod def render_templates(cls, pxifiles): for pxifile in pxifiles: From 50868997d0c4e111d75bbc9923b8ce11849c8b8a Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 28 Feb 2022 09:47:47 +0100 Subject: [PATCH 02/75] Switch order of name: to be less confusing --- .github/workflows/posix.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 158b7d578bc0c..5afa36c708ca7 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -31,38 +31,38 @@ jobs: # even if tests are skipped/xfailed pyarrow_version: ["5", "7"] include: - - env_file: actions-38-downstream_compat.yaml + - name: "Downstream Compat" + env_file: actions-38-downstream_compat.yaml pattern: "not slow and not network and not single_cpu" pytest_target: "pandas/tests/test_downstream.py" - name: "Downstream Compat" - - env_file: actions-38-minimum_versions.yaml + - name: "Minimum Versions" + env_file: actions-38-minimum_versions.yaml pattern: "not slow and not network and not single_cpu" - name: "Minimum Versions" - - env_file: actions-38.yaml + - name: "Locale: it_IT.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-it" lang: "it_IT.utf8" lc_all: "it_IT.utf8" - name: "Locale: it_IT.utf8" - - env_file: actions-38.yaml + - name: "Locale: zh_CN.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-zh-hans" lang: "zh_CN.utf8" lc_all: "zh_CN.utf8" - name: "Locale: zh_CN.utf8" - - env_file: actions-38.yaml + - name: "Data Manager" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" pandas_data_manager: "array" - name: "Data Manager" - - env_file: actions-pypy-38.yaml + - name: "Pypy" + env_file: actions-pypy-38.yaml pattern: "not slow and not network and not single_cpu" test_args: "--max-worker-restart 0" - name: "Pypy" - - env_file: actions-310-numpydev.yaml + - name: "Numpy Dev" + env_file: actions-310-numpydev.yaml pattern: "not slow and not network and not single_cpu" pandas_testing_mode: "deprecate" test_args: "-W error" - name: "Numpy Dev" fail-fast: false name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} env: From df1d92e2ff07fcd1a5fe67cd9e6cf65a9c3e0bce Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 28 Feb 2022 09:52:10 +0100 Subject: [PATCH 03/75] Fix continue condition --- .github/workflows/code-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index f097c4d53f99e..4e53961e9784e 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -82,9 +82,11 @@ jobs: - name: Run typing validation run: ci/code_checks.sh typing + if: ${{ steps.build.outcome == 'success' }} - name: Run docstring validation script tests run: pytest scripts + if: ${{ steps.build.outcome == 'success' }} asv-benchmarks: name: ASV Benchmarks From 11a67a3a5a18e556f79df7b3e63c0327e803e162 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 11:12:23 +0100 Subject: [PATCH 04/75] Squash --- .circleci/config.yml | 21 ------ .github/workflows/assign.yml | 1 + .github/workflows/asv-bot.yml | 1 + .../autoupdate-pre-commit-config.yml | 1 + .github/workflows/code-checks.yml | 4 + .github/workflows/comment_bot.yml | 1 + .github/workflows/docbuild-and-upload.yml | 1 + .github/workflows/posix.yml | 36 +-------- .github/workflows/python-dev.yml | 1 + .github/workflows/sdist.yml | 1 + azure-pipelines.yml | 56 +++++++------- ci/azure/posix.yml | 74 ++++++++++++++++--- ci/run_tests.sh | 1 + ci/setup_env.sh | 7 +- 14 files changed, 108 insertions(+), 98 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a16492b022264..e2c900cbab1d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,22 +1 @@ version: 2.1 - -jobs: - test-arm: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.medium - environment: - ENV_FILE: ci/deps/circle-38-arm64.yaml - PYTEST_WORKERS: auto - PATTERN: "not slow and not network and not clipboard and not arm_slow" - PYTEST_TARGET: "pandas" - PANDAS_CI: "1" - steps: - - checkout - - run: ci/setup_env.sh - - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh - -workflows: - test: - jobs: - - test-arm diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml index a1812843b1a8f..d8ce7a2e3564e 100644 --- a/.github/workflows/assign.yml +++ b/.github/workflows/assign.yml @@ -7,6 +7,7 @@ jobs: issue_assign: runs-on: ubuntu-latest steps: + - run: false - if: github.event.comment.body == 'take' run: | echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index 36168bee50687..5405b58b58d1c 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -27,6 +27,7 @@ jobs: cancel-in-progress: false steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: diff --git a/.github/workflows/autoupdate-pre-commit-config.yml b/.github/workflows/autoupdate-pre-commit-config.yml index 3696cba8cf2e6..2a9fcd920204b 100644 --- a/.github/workflows/autoupdate-pre-commit-config.yml +++ b/.github/workflows/autoupdate-pre-commit-config.yml @@ -11,6 +11,7 @@ jobs: name: Autoupdate pre-commit config runs-on: ubuntu-latest steps: + - run: false - name: Set up Python uses: actions/setup-python@v2 - name: Cache multiple paths diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 4e53961e9784e..53b69513b39e4 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -22,6 +22,7 @@ jobs: group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 @@ -46,6 +47,7 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: @@ -101,6 +103,7 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: @@ -144,6 +147,7 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Clean up dangling images run: docker image prune -f diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 8f610fd5781ef..0597d3fe70f0f 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -12,6 +12,7 @@ jobs: if: startsWith(github.event.comment.body, '@github-actions pre-commit') runs-on: ubuntu-latest steps: + - run: false - uses: actions/checkout@v2 - uses: r-lib/actions/pr-fetch@v2 with: diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 1d8d171078c07..c98a040c0e4c8 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -27,6 +27,7 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 5afa36c708ca7..0f43bfd19e547 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -84,43 +84,9 @@ jobs: group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }} cancel-in-progress: true - services: - mysql: - image: mysql - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: pandas - options: >- - --health-cmd "mysqladmin ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 3306:3306 - - postgres: - image: postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: pandas - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 3ec5aaf9372b9..be19e5bfe48e0 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -45,6 +45,7 @@ jobs: cancel-in-progress: true steps: + - run: false - uses: actions/checkout@v2 with: fetch-depth: 0 diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 71315947bd4f0..9d0ad6b1812dd 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -32,6 +32,7 @@ jobs: cancel-in-progress: true steps: + - run: false - uses: actions/checkout@v2 with: fetch-depth: 0 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e6dc98179b7d6..0fed801c93f5b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,33 +27,33 @@ jobs: name: macOS vmImage: macOS-10.15 -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: windows-2019 - -- job: py38_32bit - pool: - vmImage: ubuntu-18.04 +# - template: ci/azure/windows.yml +# parameters: +# name: Windows +# vmImage: windows-2019 - steps: - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - script: | - docker pull quay.io/pypa/manylinux2014_i686 - docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ - /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ - . ~/virtualenvs/pandas-dev/bin/activate && \ - python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ - python setup.py build_ext -q -j2 && \ - python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" - displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' +# - job: py38_32bit +# pool: +# vmImage: ubuntu-18.04 - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' +# steps: +# # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +# - script: | +# docker pull quay.io/pypa/manylinux2014_i686 +# docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ +# /bin/bash -xc "cd pandas && \ +# /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ +# . ~/virtualenvs/pandas-dev/bin/activate && \ +# python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ +# pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ +# python setup.py build_ext -q -j2 && \ +# python -m pip install --no-build-isolation -e . && \ +# pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" +# displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' +# +# - task: PublishTestResults@2 +# condition: succeededOrFailed() +# inputs: +# testResultsFiles: '**/test-*.xml' +# failTaskOnFailedTests: true +# testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index df1d5049be33d..c2e3b2766e0bc 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -9,35 +9,87 @@ jobs: vmImage: ${{ parameters.vmImage }} strategy: matrix: - py38: - ENV_FILE: ci/deps/actions-38.yaml - CONDA_PY: "38" + # py38: + # ENV_FILE: ci/deps/actions-38.yaml + # CONDA_PY: "38" - py39: - ENV_FILE: ci/deps/actions-39.yaml - CONDA_PY: "39" + # py39: + # ENV_FILE: ci/deps/actions-39.yaml + # CONDA_PY: "39" py310: ENV_FILE: ci/deps/actions-310.yaml CONDA_PY: "310" steps: - - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - displayName: 'Set conda path' + - task: Cache@2 + inputs: + key: sccache-0 + restoreKeys: | + sccache-restore-0 + path: $(HOME)/.sccache + displayName: Restore Sccache cache + + - script: | + set -eux + sccache_version=v0.2.15 + sccache_archive_name=sccache-$sccache_version-x86_64-apple-darwin + curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ + | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache + chmod +x /usr/local/bin/sccache + SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(HOME)/.sccache sccache --start-server + sccache -s + sccache -z + displayName: Setup Sccache + + - task: Cache@2 + inputs: + key: conda-0 + restoreKeys: | + conda-restore-0 + pattern: /usr/local/miniconda/pkgs/*.tar.bz2 + displayName: Restore Conda package cache + + - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' + displayName: 'Set conda path asdf' - script: rm /usr/local/miniconda/pkgs/cache/*.json displayName: 'Workaround for mamba-org/mamba#488' - - script: ci/setup_env.sh + - script: | + set -eux + conda install -y -c conda-forge -n base 'mamba>=0.22' pip + env | grep pkg + # See https://github.com/mamba-org/mamba/issues/633 + mamba create -q -n pandas-dev + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + echo ' - setuptools <60' >> ${ENV_FILE} + if [ "$(uname)" == Darwin ]; then + # From pyarrow on MacOS + # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib + # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib + # Reason: image not found + sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} + fi + time mamba env update -n pandas-dev --file="${ENV_FILE}" + set +e + echo Have qt? + mamba list -n pandas-dev -f qt --json + echo Have pandas? + mamba run --no-capture-output -n pandas-dev pip list | grep pandas + set -e + time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -q -j3 + time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . + sccache -s displayName: 'Setup environment and build pandas' - script: | - conda run -n pandas-dev --no-capture-output ci/run_tests.sh + mamba run --no-capture-output -n pandas-dev ci/run_tests.sh displayName: 'Test' - script: | pushd /tmp - conda run -n pandas-dev python -c "import pandas; pandas.show_versions()" + mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" popd displayName: 'Build versions' diff --git a/ci/run_tests.sh b/ci/run_tests.sh index e6de5caf955fc..245155515921f 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,4 +1,5 @@ #!/bin/bash -e +exit 0 # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 diff --git a/ci/setup_env.sh b/ci/setup_env.sh index a85767eb6f1b4..15da3905cebad 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -14,6 +14,8 @@ fi MINICONDA_DIR=/usr/local/miniconda +find /usr | grep miniconda | head +find /home | grep miniconda | head if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then echo "Found Miniconda installation at $MINICONDA_DIR" else @@ -34,12 +36,12 @@ else exit 1 fi echo "Downloading $CONDA_URL" - wget -q $CONDA_URL -O miniconda.sh + time wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh MINICONDA_DIR="$HOME/miniconda3" rm -rf $MINICONDA_DIR - ./miniconda.sh -b -p $MINICONDA_DIR + time ./miniconda.sh -b -p $MINICONDA_DIR fi export PATH=$MINICONDA_DIR/bin:$PATH @@ -51,7 +53,6 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip echo "conda info -a" From 39d57b17a2117c4ccd0de1af37dd33e04e2b98bc Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 13:12:30 +0100 Subject: [PATCH 05/75] empty From bf9d342afeb2906d909abedc401e3ee98a2d104c Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 13:28:05 +0100 Subject: [PATCH 06/75] WIP --- ci/azure/posix.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c2e3b2766e0bc..59f465ce450e0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,8 +50,11 @@ jobs: pattern: /usr/local/miniconda/pkgs/*.tar.bz2 displayName: Restore Conda package cache + - script: ls -lh /usr/local/miniconda/pkgs + displayName: 'List packages' + - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' - displayName: 'Set conda path asdf' + displayName: 'Set conda path' - script: rm /usr/local/miniconda/pkgs/cache/*.json displayName: 'Workaround for mamba-org/mamba#488' @@ -78,7 +81,7 @@ jobs: echo Have pandas? mamba run --no-capture-output -n pandas-dev pip list | grep pandas set -e - time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -q -j3 + time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . sccache -s displayName: 'Setup environment and build pandas' From 4833ee50b4d456a2d42e94ed5993c61ce5ab3629 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 13:43:42 +0100 Subject: [PATCH 07/75] WIP --- ci/azure/posix.yml | 110 ++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 59f465ce450e0..99ab482c3aa82 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -42,64 +42,72 @@ jobs: sccache -z displayName: Setup Sccache - - task: Cache@2 - inputs: - key: conda-0 - restoreKeys: | - conda-restore-0 - pattern: /usr/local/miniconda/pkgs/*.tar.bz2 - displayName: Restore Conda package cache - - script: ls -lh /usr/local/miniconda/pkgs displayName: 'List packages' + # - task: Cache@2 + # inputs: + # key: conda-0 + # restoreKeys: | + # conda-restore-0 + # pattern: | + # /usr/local/miniconda/pkgs/*.conda + # /usr/local/miniconda/pkgs/*.tar.bz2 + # displayName: Restore Conda package cache + + # - script: ls -lh /usr/local/miniconda/pkgs + # displayName: 'List packages' + - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' - - script: rm /usr/local/miniconda/pkgs/cache/*.json - displayName: 'Workaround for mamba-org/mamba#488' + - script: conda env list + displayName: 'conda list' - - script: | - set -eux - conda install -y -c conda-forge -n base 'mamba>=0.22' pip - env | grep pkg - # See https://github.com/mamba-org/mamba/issues/633 - mamba create -q -n pandas-dev - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - echo ' - setuptools <60' >> ${ENV_FILE} - if [ "$(uname)" == Darwin ]; then - # From pyarrow on MacOS - # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib - # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib - # Reason: image not found - sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} - fi - time mamba env update -n pandas-dev --file="${ENV_FILE}" - set +e - echo Have qt? - mamba list -n pandas-dev -f qt --json - echo Have pandas? - mamba run --no-capture-output -n pandas-dev pip list | grep pandas - set -e - time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . - sccache -s - displayName: 'Setup environment and build pandas' +# - script: rm /usr/local/miniconda/pkgs/cache/*.json +# displayName: 'Workaround for mamba-org/mamba#488' - - script: | - mamba run --no-capture-output -n pandas-dev ci/run_tests.sh - displayName: 'Test' +# - script: | +# set -eux +# conda install -y -c conda-forge -n base 'mamba>=0.22' pip +# env | grep pkg +# # See https://github.com/mamba-org/mamba/issues/633 +# mamba create -q -n pandas-dev +# # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +# echo ' - setuptools <60' >> ${ENV_FILE} +# if [ "$(uname)" == Darwin ]; then +# # From pyarrow on MacOS +# # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib +# # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib +# # Reason: image not found +# sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} +# fi +# time mamba env update -n pandas-dev --file="${ENV_FILE}" +# set +e +# echo Have qt? +# mamba list -n pandas-dev -f qt --json +# echo Have pandas? +# mamba run --no-capture-output -n pandas-dev pip list | grep pandas +# set -e +# time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 +# time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . +# sccache -s +# displayName: 'Setup environment and build pandas' - - script: | - pushd /tmp - mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" - popd - displayName: 'Build versions' +# - script: | +# mamba run --no-capture-output -n pandas-dev ci/run_tests.sh +# displayName: 'Test' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - failTaskOnFailedTests: true - testResultsFiles: 'test-data.xml' - testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} - displayName: 'Publish test results' +# - script: | +# pushd /tmp +# mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" +# popd +# displayName: 'Build versions' + +# - task: PublishTestResults@2 +# condition: succeededOrFailed() +# inputs: +# failTaskOnFailedTests: true +# testResultsFiles: 'test-data.xml' +# testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} +# displayName: 'Publish test results' From 5dce1339da550bbc1ea6f70346c1d0f9b4021d7a Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 13:46:17 +0100 Subject: [PATCH 08/75] empty From 1bd8f5405f93a65e9961b1826fbfa5df16a453c3 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 13:53:01 +0100 Subject: [PATCH 09/75] WIP --- ci/azure/posix.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 99ab482c3aa82..eba4c5d34edab 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -45,18 +45,18 @@ jobs: - script: ls -lh /usr/local/miniconda/pkgs displayName: 'List packages' - # - task: Cache@2 - # inputs: - # key: conda-0 - # restoreKeys: | - # conda-restore-0 - # pattern: | - # /usr/local/miniconda/pkgs/*.conda - # /usr/local/miniconda/pkgs/*.tar.bz2 - # displayName: Restore Conda package cache + - task: Cache@2 + inputs: + key: conda-0 + restoreKeys: | + conda-restore-0 + pattern: | + /usr/local/miniconda/pkgs/*.conda + /usr/local/miniconda/pkgs/*.tar.bz2 + displayName: Restore Conda package cache - # - script: ls -lh /usr/local/miniconda/pkgs - # displayName: 'List packages' + - script: ls -lh /usr/local/miniconda/pkgs + displayName: 'List packages' - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' From de04d242d3e3f1b37c6a36c67463c3929b03a245 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 13:58:39 +0100 Subject: [PATCH 10/75] WIP --- ci/azure/posix.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index eba4c5d34edab..4eeef4376dd5e 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,12 +50,18 @@ jobs: key: conda-0 restoreKeys: | conda-restore-0 + path: /usr/local/miniconda pattern: | - /usr/local/miniconda/pkgs/*.conda - /usr/local/miniconda/pkgs/*.tar.bz2 + *.conda + *.tar.bz2 displayName: Restore Conda package cache - - script: ls -lh /usr/local/miniconda/pkgs + - script: | + ls -lh /usr/local/miniconda/pkgs + ls / + ls ~ + ls + ls /usr/local/miniconda displayName: 'List packages' - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' @@ -64,6 +70,9 @@ jobs: - script: conda env list displayName: 'conda list' + - script: conda install -n base werkzeug + display: test + # - script: rm /usr/local/miniconda/pkgs/cache/*.json # displayName: 'Workaround for mamba-org/mamba#488' From 8560814d20e6d7cdbb6a8d33b004ed10db211e49 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:04:15 +0100 Subject: [PATCH 11/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 4eeef4376dd5e..b09936fe6065d 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -71,7 +71,7 @@ jobs: displayName: 'conda list' - script: conda install -n base werkzeug - display: test + displayName: test # - script: rm /usr/local/miniconda/pkgs/cache/*.json # displayName: 'Workaround for mamba-org/mamba#488' From ac418a2dad42e4fe9325d9b5c7149e16f0d0d8a8 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:14:40 +0100 Subject: [PATCH 12/75] WIP --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index b09936fe6065d..8a639cdd7f132 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,9 +47,9 @@ jobs: - task: Cache@2 inputs: - key: conda-0 + key: conda-1 restoreKeys: | - conda-restore-0 + conda-restore-1 path: /usr/local/miniconda pattern: | *.conda From 889118921ef2ceb376469ac838f7f6506615accb Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:18:28 +0100 Subject: [PATCH 13/75] empty From 6ced933086ed14a1f852e030cbfa3a6af7a8cbec Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:21:39 +0100 Subject: [PATCH 14/75] WIP --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 8a639cdd7f132..46aa7886e5957 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -52,8 +52,8 @@ jobs: conda-restore-1 path: /usr/local/miniconda pattern: | - *.conda - *.tar.bz2 + **/*.conda + **/*.tar.bz2 displayName: Restore Conda package cache - script: | From 6c3e544418e13c44b896c09e20cfafb027bd7533 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:23:57 +0100 Subject: [PATCH 15/75] WIP --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 46aa7886e5957..2f4ad7322a4ea 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,9 +47,9 @@ jobs: - task: Cache@2 inputs: - key: conda-1 + key: conda-2 restoreKeys: | - conda-restore-1 + conda-restore-2 path: /usr/local/miniconda pattern: | **/*.conda From 92d39357ca64ecce3e0fb6e6a448bcc3cbdd2742 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:35:40 +0100 Subject: [PATCH 16/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 2f4ad7322a4ea..cd7fd4facceca 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -70,7 +70,7 @@ jobs: - script: conda env list displayName: 'conda list' - - script: conda install -n base werkzeug + - script: conda install -c conda-forge -n base werkzeug displayName: test # - script: rm /usr/local/miniconda/pkgs/cache/*.json From 73e27f07e718cafb2b1dbc65aa60db76a66a5c21 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:44:16 +0100 Subject: [PATCH 17/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index cd7fd4facceca..79765c0a8de63 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -70,7 +70,7 @@ jobs: - script: conda env list displayName: 'conda list' - - script: conda install -c conda-forge -n base werkzeug + - script: conda install -c conda-forge -n base werkzeug -y displayName: test # - script: rm /usr/local/miniconda/pkgs/cache/*.json From 665e68913f8af26a5272e2e7d083e561675c6a08 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 14:57:41 +0100 Subject: [PATCH 18/75] WIP --- ci/azure/posix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 79765c0a8de63..ab7defbcbc452 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,10 +47,10 @@ jobs: - task: Cache@2 inputs: - key: conda-2 + key: conda-3 restoreKeys: | - conda-restore-2 - path: /usr/local/miniconda + conda-restore-3 + path: /usr/local/miniconda/pkgs pattern: | **/*.conda **/*.tar.bz2 From b5e0fa2550ea19a0e08b5daa61d40cdd7f05bfe0 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:09:20 +0100 Subject: [PATCH 19/75] WIP --- ci/azure/posix.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index ab7defbcbc452..db7c7cf4b2429 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,13 +47,11 @@ jobs: - task: Cache@2 inputs: - key: conda-3 + key: conda-4 restoreKeys: | - conda-restore-3 + conda-restore-4 path: /usr/local/miniconda/pkgs - pattern: | - **/*.conda - **/*.tar.bz2 + pattern: /usr/local/miniconda/pkgs/*.tar.bz2 displayName: Restore Conda package cache - script: | From 1172fa24ee4823cc0db508fe8adde05f17410c18 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:14:04 +0100 Subject: [PATCH 20/75] empty From a51621ce722bb3323a146b961e5531775e237985 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:14:30 +0100 Subject: [PATCH 21/75] WIP --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index db7c7cf4b2429..050e5b302684d 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,7 +28,7 @@ jobs: restoreKeys: | sccache-restore-0 path: $(HOME)/.sccache - displayName: Restore Sccache cache + displayName: Sccache cache - script: | set -eux @@ -52,7 +52,7 @@ jobs: conda-restore-4 path: /usr/local/miniconda/pkgs pattern: /usr/local/miniconda/pkgs/*.tar.bz2 - displayName: Restore Conda package cache + displayName: Conda download cache - script: | ls -lh /usr/local/miniconda/pkgs From 61a8785fff16cecb41eabb20851e6b89a027afd3 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:25:43 +0100 Subject: [PATCH 22/75] WIP --- ci/azure/posix.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 050e5b302684d..ff820e52bc8e7 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,11 +47,10 @@ jobs: - task: Cache@2 inputs: - key: conda-4 + key: conda-5 restoreKeys: | - conda-restore-4 + conda-restore-5 path: /usr/local/miniconda/pkgs - pattern: /usr/local/miniconda/pkgs/*.tar.bz2 displayName: Conda download cache - script: | @@ -118,3 +117,6 @@ jobs: # testResultsFiles: 'test-data.xml' # testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} # displayName: 'Publish test results' + + - script: conda clean -yf + displayName: 'Reduce Conda cache size' From 17143d898ba4334964dc9634654a6bd0eda0c543 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:29:32 +0100 Subject: [PATCH 23/75] WIP --- ci/azure/posix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index ff820e52bc8e7..5ff530eaa5bb7 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,9 +47,9 @@ jobs: - task: Cache@2 inputs: - key: conda-5 + key: conda-6 restoreKeys: | - conda-restore-5 + conda-restore-6 path: /usr/local/miniconda/pkgs displayName: Conda download cache @@ -118,5 +118,5 @@ jobs: # testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} # displayName: 'Publish test results' - - script: conda clean -yf + - script: conda clean -yp displayName: 'Reduce Conda cache size' From 46417203d9b4384d2eca8a34bf8b93059ef57262 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:40:41 +0100 Subject: [PATCH 24/75] WIP --- ci/azure/posix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 5ff530eaa5bb7..1f762653a9480 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -47,9 +47,9 @@ jobs: - task: Cache@2 inputs: - key: conda-6 + key: conda-7 restoreKeys: | - conda-restore-6 + conda-restore-7 path: /usr/local/miniconda/pkgs displayName: Conda download cache @@ -118,5 +118,5 @@ jobs: # testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} # displayName: 'Publish test results' - - script: conda clean -yp + - script: find /usr/local/miniconda/pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r displayName: 'Reduce Conda cache size' From e695d121dd4ea0b9e932017789f8d45883b2cd93 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:45:25 +0100 Subject: [PATCH 25/75] WIP --- ci/azure/posix.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 1f762653a9480..095aae1bfa2cc 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -118,5 +118,7 @@ jobs: # testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} # displayName: 'Publish test results' - - script: find /usr/local/miniconda/pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r + - script: | + find /usr/local/miniconda/pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r + ls /usr/local/miniconda/pkgs displayName: 'Reduce Conda cache size' From 656ec7f1635c980bb75d450ab7c38350e608ba4f Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:51:07 +0100 Subject: [PATCH 26/75] WIP --- ci/azure/posix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 095aae1bfa2cc..2abc0249e4a88 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,6 +22,9 @@ jobs: CONDA_PY: "310" steps: + - script: rm /usr/local/miniconda/pkgs/cache/*.json + displayName: 'Workaround for mamba-org/mamba#488' + - task: Cache@2 inputs: key: sccache-0 @@ -70,9 +73,6 @@ jobs: - script: conda install -c conda-forge -n base werkzeug -y displayName: test -# - script: rm /usr/local/miniconda/pkgs/cache/*.json -# displayName: 'Workaround for mamba-org/mamba#488' - # - script: | # set -eux # conda install -y -c conda-forge -n base 'mamba>=0.22' pip From 3a21ba6b5ecb0e1cfbc885adf02367807cbf5e98 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 15:56:44 +0100 Subject: [PATCH 27/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 2abc0249e4a88..adc9ddd52b3da 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,7 +22,7 @@ jobs: CONDA_PY: "310" steps: - - script: rm /usr/local/miniconda/pkgs/cache/*.json + - script: rm -r /usr/local/miniconda/pkgs/cache displayName: 'Workaround for mamba-org/mamba#488' - task: Cache@2 From 68e6768ee36d6986f098eb497cc0e54bd13bb446 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:07:35 +0100 Subject: [PATCH 28/75] WIP --- ci/azure/posix.yml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index adc9ddd52b3da..c32ecf19f9570 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,7 +22,7 @@ jobs: CONDA_PY: "310" steps: - - script: rm -r /usr/local/miniconda/pkgs/cache + - script: rm /usr/local/miniconda/pkgs/cache/*.json displayName: 'Workaround for mamba-org/mamba#488' - task: Cache@2 @@ -50,30 +50,25 @@ jobs: - task: Cache@2 inputs: - key: conda-7 + key: conda-8 restoreKeys: | - conda-restore-7 - path: /usr/local/miniconda/pkgs + conda-restore-8 + path: ~/conda_pkgs displayName: Conda download cache - - script: | - ls -lh /usr/local/miniconda/pkgs - ls / - ls ~ - ls - ls /usr/local/miniconda - displayName: 'List packages' - - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' - script: conda env list displayName: 'conda list' - - script: conda install -c conda-forge -n base werkzeug -y + - script: | + export CONDA_PKGS_DIR=~/conda_pkgs + conda install -c conda-forge -n base werkzeug -y displayName: test # - script: | +# export CONDA_PKGS_DIR=~/conda_pkgs # set -eux # conda install -y -c conda-forge -n base 'mamba>=0.22' pip # env | grep pkg @@ -119,6 +114,6 @@ jobs: # displayName: 'Publish test results' - script: | - find /usr/local/miniconda/pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r - ls /usr/local/miniconda/pkgs + find ~/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r + ls ~/conda_pkgs displayName: 'Reduce Conda cache size' From fac399dec3f464856d392481f9b88565c6d16687 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:09:38 +0100 Subject: [PATCH 29/75] WIP --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c32ecf19f9570..de6b2b3961a13 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -63,8 +63,8 @@ jobs: displayName: 'conda list' - script: | - export CONDA_PKGS_DIR=~/conda_pkgs - conda install -c conda-forge -n base werkzeug -y + export CONDA_PKGS_DIR=~/conda_pkgs + conda install -c conda-forge -n base werkzeug -y displayName: test # - script: | From 1ef0e08f4104de1af6ea0c8cf59167ed7651ba8a Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:14:15 +0100 Subject: [PATCH 30/75] WIP --- ci/azure/posix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index de6b2b3961a13..98cd04157212f 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -63,6 +63,7 @@ jobs: displayName: 'conda list' - script: | + mkdir -p ~/conda_pkgs export CONDA_PKGS_DIR=~/conda_pkgs conda install -c conda-forge -n base werkzeug -y displayName: test From fe6b1d6fa61628620a80644522e57f6740a1b3b7 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:20:01 +0100 Subject: [PATCH 31/75] WIP --- ci/azure/posix.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 98cd04157212f..537af33ba798c 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,10 +50,10 @@ jobs: - task: Cache@2 inputs: - key: conda-8 + key: conda-9 restoreKeys: | - conda-restore-8 - path: ~/conda_pkgs + conda-restore-9 + path: $(HOME)/conda_pkgs displayName: Conda download cache - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' @@ -63,13 +63,13 @@ jobs: displayName: 'conda list' - script: | - mkdir -p ~/conda_pkgs - export CONDA_PKGS_DIR=~/conda_pkgs + mkdir -p $(HOME)/conda_pkgs + export CONDA_PKGS_DIR=$(HOME)/conda_pkgs conda install -c conda-forge -n base werkzeug -y displayName: test # - script: | -# export CONDA_PKGS_DIR=~/conda_pkgs +# export CONDA_PKGS_DIR=$(HOME)/conda_pkgs # set -eux # conda install -y -c conda-forge -n base 'mamba>=0.22' pip # env | grep pkg @@ -115,6 +115,6 @@ jobs: # displayName: 'Publish test results' - script: | - find ~/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r - ls ~/conda_pkgs + find $(HOME)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r + ls $(HOME)/conda_pkgs displayName: 'Reduce Conda cache size' From aea339d0cdc0b3e7e6f16d3654036e5e96ed323d Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:25:39 +0100 Subject: [PATCH 32/75] WIP --- ci/azure/posix.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 537af33ba798c..80eee12da598a 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,9 +50,9 @@ jobs: - task: Cache@2 inputs: - key: conda-9 + key: conda-10 restoreKeys: | - conda-restore-9 + conda-restore-10 path: $(HOME)/conda_pkgs displayName: Conda download cache @@ -64,12 +64,12 @@ jobs: - script: | mkdir -p $(HOME)/conda_pkgs - export CONDA_PKGS_DIR=$(HOME)/conda_pkgs + export CONDA_PKGS_DIRS=$(HOME)/conda_pkgs conda install -c conda-forge -n base werkzeug -y displayName: test # - script: | -# export CONDA_PKGS_DIR=$(HOME)/conda_pkgs +# export CONDA_PKGS_DIRS=$(HOME)/conda_pkgs # set -eux # conda install -y -c conda-forge -n base 'mamba>=0.22' pip # env | grep pkg From ed61cfa9a6534607476a6bed49226b4a82340680 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:30:34 +0100 Subject: [PATCH 33/75] empty From 36f934702dfa52c8647574d8c22aacc1bf857085 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:38:59 +0100 Subject: [PATCH 34/75] empty From e8a17179ec59c68c2e630df0b60e86d4877a0366 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:39:03 +0100 Subject: [PATCH 35/75] WIP --- ci/azure/posix.yml | 84 ++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 80eee12da598a..c1eefc67ecf6a 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -30,7 +30,7 @@ jobs: key: sccache-0 restoreKeys: | sccache-restore-0 - path: $(HOME)/.sccache + path: $(Pipeline.Workspace)/.sccache displayName: Sccache cache - script: | @@ -40,61 +40,59 @@ jobs: curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache chmod +x /usr/local/bin/sccache - SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(HOME)/.sccache sccache --start-server + SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(Pipeline.Workspace)/.sccache sccache --start-server sccache -s sccache -z displayName: Setup Sccache - - script: ls -lh /usr/local/miniconda/pkgs + - script: ls -lh $(Pipeline.Workspace)/conda_pkgs displayName: 'List packages' + - script: conda env list + displayName: 'conda list' + - task: Cache@2 inputs: - key: conda-10 + key: conda-11 restoreKeys: | - conda-restore-10 - path: $(HOME)/conda_pkgs + conda-restore-11 + path: $(Pipeline.Workspace)/conda_pkgs + cacheHitVar: CONDA_CACHE_RESTORED displayName: Conda download cache + - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs + displayName: Create Conda pkgs directory + condition: eq(variables.CONDA_CACHE_RESTORED, 'true') + - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' - - script: conda env list - displayName: 'conda list' - - script: | - mkdir -p $(HOME)/conda_pkgs - export CONDA_PKGS_DIRS=$(HOME)/conda_pkgs - conda install -c conda-forge -n base werkzeug -y - displayName: test - -# - script: | -# export CONDA_PKGS_DIRS=$(HOME)/conda_pkgs -# set -eux -# conda install -y -c conda-forge -n base 'mamba>=0.22' pip -# env | grep pkg -# # See https://github.com/mamba-org/mamba/issues/633 -# mamba create -q -n pandas-dev -# # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 -# echo ' - setuptools <60' >> ${ENV_FILE} -# if [ "$(uname)" == Darwin ]; then -# # From pyarrow on MacOS -# # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib -# # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib -# # Reason: image not found -# sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} -# fi -# time mamba env update -n pandas-dev --file="${ENV_FILE}" -# set +e -# echo Have qt? -# mamba list -n pandas-dev -f qt --json -# echo Have pandas? -# mamba run --no-capture-output -n pandas-dev pip list | grep pandas -# set -e -# time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 -# time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . -# sccache -s -# displayName: 'Setup environment and build pandas' + set -eux + export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs + conda install -y -c conda-forge -n base 'mamba>=0.22' pip + # See https://github.com/mamba-org/mamba/issues/633 + mamba create -q -n pandas-dev + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + echo ' - setuptools <60' >> ${ENV_FILE} + if [ "$(uname)" == Darwin ]; then + # From pyarrow on MacOS + # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib + # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib + # Reason: image not found + sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} + fi + time mamba env update -n pandas-dev --file="${ENV_FILE}" + set +e + echo Have qt? + mamba list -n pandas-dev -f qt --json + echo Have pandas? + mamba run --no-capture-output -n pandas-dev pip list | grep pandas + set -e + time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 + time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . + sccache -s + displayName: 'Setup environment and build pandas' # - script: | # mamba run --no-capture-output -n pandas-dev ci/run_tests.sh @@ -115,6 +113,6 @@ jobs: # displayName: 'Publish test results' - script: | - find $(HOME)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r - ls $(HOME)/conda_pkgs + find $(Pipeline.Workspace)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r + ls $(Pipeline.Workspace)/conda_pkgs displayName: 'Reduce Conda cache size' From 8c3d853240489bdc511157403150e632a68c4b36 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 16:41:32 +0100 Subject: [PATCH 36/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c1eefc67ecf6a..de2144c1141b3 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -45,7 +45,7 @@ jobs: sccache -z displayName: Setup Sccache - - script: ls -lh $(Pipeline.Workspace)/conda_pkgs + - script: ls -lh $(Pipeline.Workspace)/conda_pkgs || true displayName: 'List packages' - script: conda env list From ef4fa89914327b07786454d4a3979aeca14c1959 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 18:47:16 +0100 Subject: [PATCH 37/75] empty From b36f522268a4335024579874120ae8078ffb4aba Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:02:40 +0100 Subject: [PATCH 38/75] WIP --- ci/azure/posix.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index de2144c1141b3..9cb25b58d5d12 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -62,17 +62,18 @@ jobs: - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs displayName: Create Conda pkgs directory - condition: eq(variables.CONDA_CACHE_RESTORED, 'true') + condition: ne(variables.CONDA_CACHE_RESTORED, 'true') - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' - script: | set -eux + ls -lh export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs - conda install -y -c conda-forge -n base 'mamba>=0.22' pip + time conda install -y -c conda-forge -n base 'mamba>=0.22' pip # See https://github.com/mamba-org/mamba/issues/633 - mamba create -q -n pandas-dev + time mamba create -q -n pandas-dev # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 echo ' - setuptools <60' >> ${ENV_FILE} if [ "$(uname)" == Darwin ]; then @@ -116,3 +117,4 @@ jobs: find $(Pipeline.Workspace)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r ls $(Pipeline.Workspace)/conda_pkgs displayName: 'Reduce Conda cache size' + condition: ne(variables.CONDA_CACHE_RESTORED, 'true') From 70c8e5369dd3ab24262f63b5c5b04d7c5277965b Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:17:38 +0100 Subject: [PATCH 39/75] WIP --- ci/azure/posix.yml | 97 ++++++---------------------------------------- 1 file changed, 11 insertions(+), 86 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9cb25b58d5d12..33f26e9a31caa 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,99 +22,24 @@ jobs: CONDA_PY: "310" steps: - - script: rm /usr/local/miniconda/pkgs/cache/*.json - displayName: 'Workaround for mamba-org/mamba#488' - - task: Cache@2 inputs: - key: sccache-0 + key: mamba-0 restoreKeys: | - sccache-restore-0 - path: $(Pipeline.Workspace)/.sccache - displayName: Sccache cache - - - script: | - set -eux - sccache_version=v0.2.15 - sccache_archive_name=sccache-$sccache_version-x86_64-apple-darwin - curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ - | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache - chmod +x /usr/local/bin/sccache - SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(Pipeline.Workspace)/.sccache sccache --start-server - sccache -s - sccache -z - displayName: Setup Sccache - - - script: ls -lh $(Pipeline.Workspace)/conda_pkgs || true - displayName: 'List packages' - - - script: conda env list - displayName: 'conda list' - - - task: Cache@2 - inputs: - key: conda-11 - restoreKeys: | - conda-restore-11 - path: $(Pipeline.Workspace)/conda_pkgs - cacheHitVar: CONDA_CACHE_RESTORED - displayName: Conda download cache - - - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs - displayName: Create Conda pkgs directory - condition: ne(variables.CONDA_CACHE_RESTORED, 'true') - - - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' - displayName: 'Set conda path' + mamba-restore-0 + path: /usr/local/miniconda/envs/mamba + cacheHitVar: CONDA_MAMBA_ENV_CACHE_RESTORED + displayName: Conda Mamba env cache - script: | set -eux ls -lh + ls -lh $(Pipeline.Workspace) export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs - time conda install -y -c conda-forge -n base 'mamba>=0.22' pip - # See https://github.com/mamba-org/mamba/issues/633 - time mamba create -q -n pandas-dev - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - echo ' - setuptools <60' >> ${ENV_FILE} - if [ "$(uname)" == Darwin ]; then - # From pyarrow on MacOS - # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib - # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib - # Reason: image not found - sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} - fi - time mamba env update -n pandas-dev --file="${ENV_FILE}" - set +e - echo Have qt? - mamba list -n pandas-dev -f qt --json - echo Have pandas? - mamba run --no-capture-output -n pandas-dev pip list | grep pandas - set -e - time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . - sccache -s - displayName: 'Setup environment and build pandas' - -# - script: | -# mamba run --no-capture-output -n pandas-dev ci/run_tests.sh -# displayName: 'Test' - -# - script: | -# pushd /tmp -# mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" -# popd -# displayName: 'Build versions' - -# - task: PublishTestResults@2 -# condition: succeededOrFailed() -# inputs: -# failTaskOnFailedTests: true -# testResultsFiles: 'test-data.xml' -# testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} -# displayName: 'Publish test results' + time conda install -y -n mamba -c conda-forge 'mamba>=0.22' + displayName: Install Mamba + condition: ne(variables.CONDA_MAMBA_ENV_CACHE_RESTORED, 'true') - script: | - find $(Pipeline.Workspace)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r - ls $(Pipeline.Workspace)/conda_pkgs - displayName: 'Reduce Conda cache size' - condition: ne(variables.CONDA_CACHE_RESTORED, 'true') + conda run --no-capture-output -n mamba mamba --help + display: Mamba help From 45b5df65dcf9be1cffb14a829ad7759c91b54452 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:21:00 +0100 Subject: [PATCH 40/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 33f26e9a31caa..8d09f4e9660ad 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -42,4 +42,4 @@ jobs: - script: | conda run --no-capture-output -n mamba mamba --help - display: Mamba help + displayName: Mamba help From 84d19a7edf6011ea0b7e7404fab09647f5800226 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:24:05 +0100 Subject: [PATCH 41/75] WIP --- ci/azure/posix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 8d09f4e9660ad..4c8a48596b062 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -36,6 +36,8 @@ jobs: ls -lh ls -lh $(Pipeline.Workspace) export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs + # See https://github.com/mamba-org/mamba/issues/633 + time conda create -n mamba time conda install -y -n mamba -c conda-forge 'mamba>=0.22' displayName: Install Mamba condition: ne(variables.CONDA_MAMBA_ENV_CACHE_RESTORED, 'true') From 57c552a4ac6f33b45c57fcff22c5d0753f1ab80c Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:32:14 +0100 Subject: [PATCH 42/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 4c8a48596b062..8371c800ddc69 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -37,7 +37,7 @@ jobs: ls -lh $(Pipeline.Workspace) export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs # See https://github.com/mamba-org/mamba/issues/633 - time conda create -n mamba + time conda create -y -n mamba time conda install -y -n mamba -c conda-forge 'mamba>=0.22' displayName: Install Mamba condition: ne(variables.CONDA_MAMBA_ENV_CACHE_RESTORED, 'true') From dbcd3af6ec50031769c82cfc31901030592c9700 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:37:18 +0100 Subject: [PATCH 43/75] WIP --- ci/azure/posix.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 8371c800ddc69..e0b4dc2c5dd79 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,15 +22,6 @@ jobs: CONDA_PY: "310" steps: - - task: Cache@2 - inputs: - key: mamba-0 - restoreKeys: | - mamba-restore-0 - path: /usr/local/miniconda/envs/mamba - cacheHitVar: CONDA_MAMBA_ENV_CACHE_RESTORED - displayName: Conda Mamba env cache - - script: | set -eux ls -lh @@ -43,5 +34,5 @@ jobs: condition: ne(variables.CONDA_MAMBA_ENV_CACHE_RESTORED, 'true') - script: | - conda run --no-capture-output -n mamba mamba --help + conda run --no-capture-output -n mamba mamba info displayName: Mamba help From 8e97558bb3b3d072638d5af33c6840b90e4559ef Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:40:04 +0100 Subject: [PATCH 44/75] WIP --- ci/azure/posix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index e0b4dc2c5dd79..b1d952a974db8 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -28,11 +28,11 @@ jobs: ls -lh $(Pipeline.Workspace) export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs # See https://github.com/mamba-org/mamba/issues/633 - time conda create -y -n mamba - time conda install -y -n mamba -c conda-forge 'mamba>=0.22' + #time conda create -y -n mamba + time conda install -y -n base -c conda-forge 'mamba>=0.22' displayName: Install Mamba condition: ne(variables.CONDA_MAMBA_ENV_CACHE_RESTORED, 'true') - script: | - conda run --no-capture-output -n mamba mamba info + conda run --no-capture-output -n base mamba info displayName: Mamba help From 2daa50454886f30f00b4f4bd7c92efef157bc389 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 19:49:35 +0100 Subject: [PATCH 45/75] WIP --- ci/azure/posix.yml | 104 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 6 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index b1d952a974db8..9db9115acaad0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,17 +22,109 @@ jobs: CONDA_PY: "310" steps: + - script: | + set -eux + # Workaround for mamba-org/mamba#488 + rm -f /usr/local/miniconda/pkgs/cache/*.json + # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) + conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' + displayName: Install Mamba + + - task: Cache@2 + inputs: + key: sccache-0 + restoreKeys: | + sccache-restore-0 + path: $(Pipeline.Workspace)/.sccache + displayName: Sccache cache + + - script: | + set -eux + sccache_version=v0.2.15 + sccache_archive_name=sccache-$sccache_version-x86_64-apple-darwin + curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ + | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache + chmod +x /usr/local/bin/sccache + SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(Pipeline.Workspace)/.sccache sccache --start-server + sccache -s + sccache -z + displayName: Setup Sccache + + - script: ls -lh $(Pipeline.Workspace)/conda_pkgs || true + displayName: 'List packages' + + - script: conda env list + displayName: 'conda list' + + - task: Cache@2 + inputs: + key: conda-11 + restoreKeys: | + conda-restore-11 + path: $(Pipeline.Workspace)/conda_pkgs + cacheHitVar: CONDA_CACHE_RESTORED + displayName: Conda download cache + + - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs + displayName: Create Conda pkgs directory + condition: ne(variables.CONDA_CACHE_RESTORED, 'true') + + - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' + displayName: 'Set conda path' + - script: | set -eux ls -lh ls -lh $(Pipeline.Workspace) export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs + + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + echo ' - setuptools <60' >> ${ENV_FILE} + + if [ "$(uname)" == Darwin ]; then + # From pyarrow on MacOS + # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib + # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib + # Reason: image not found + sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} + fi + # See https://github.com/mamba-org/mamba/issues/633 - #time conda create -y -n mamba - time conda install -y -n base -c conda-forge 'mamba>=0.22' - displayName: Install Mamba - condition: ne(variables.CONDA_MAMBA_ENV_CACHE_RESTORED, 'true') + time conda run --no-capture-output -n mamba-env mamba create -n pandas-dev + time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" + + set +e + echo Have qt? + conda run -n mamba-env mamba list -n pandas-dev -f qt --json + echo Have pandas? + conda run -n mamba-env mamba list -n pandas-dev -f pandas --json + conda run -n mamba-env mamba run --no-capture-output -n pandas-dev pip list | grep pandas + set -e + time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 + time mamba run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + sccache -s + displayName: 'Setup environment and build pandas' + +# - script: | +# mamba run --no-capture-output -n pandas-dev ci/run_tests.sh +# displayName: 'Test' + +# - script: | +# pushd /tmp +# mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" +# popd +# displayName: 'Build versions' + +# - task: PublishTestResults@2 +# condition: succeededOrFailed() +# inputs: +# failTaskOnFailedTests: true +# testResultsFiles: 'test-data.xml' +# testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} +# displayName: 'Publish test results' - script: | - conda run --no-capture-output -n base mamba info - displayName: Mamba help + find $(Pipeline.Workspace)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r + ls $(Pipeline.Workspace)/conda_pkgs + displayName: 'Reduce Conda cache size' + condition: ne(variables.CONDA_CACHE_RESTORED, 'true') From 59fbbc72e5489551aec2fe6f428282bfe8092094 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:08:02 +0100 Subject: [PATCH 46/75] WIP --- ci/azure/posix.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9db9115acaad0..37faf97b247fb 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -56,18 +56,18 @@ jobs: - script: conda env list displayName: 'conda list' - - task: Cache@2 - inputs: - key: conda-11 - restoreKeys: | - conda-restore-11 - path: $(Pipeline.Workspace)/conda_pkgs - cacheHitVar: CONDA_CACHE_RESTORED - displayName: Conda download cache - - - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs - displayName: Create Conda pkgs directory - condition: ne(variables.CONDA_CACHE_RESTORED, 'true') + # - task: Cache@2 + # inputs: + # key: conda-11 + # restoreKeys: | + # conda-restore-11 + # path: $(Pipeline.Workspace)/conda_pkgs + # cacheHitVar: CONDA_CACHE_RESTORED + # displayName: Conda download cache + + # - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs + # displayName: Create Conda pkgs directory + # condition: ne(variables.CONDA_CACHE_RESTORED, 'true') - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' @@ -76,7 +76,7 @@ jobs: set -eux ls -lh ls -lh $(Pipeline.Workspace) - export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs + # export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 echo ' - setuptools <60' >> ${ENV_FILE} @@ -90,28 +90,28 @@ jobs: fi # See https://github.com/mamba-org/mamba/issues/633 - time conda run --no-capture-output -n mamba-env mamba create -n pandas-dev + conda create -n pandas-dev time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" set +e echo Have qt? - conda run -n mamba-env mamba list -n pandas-dev -f qt --json + conda list -n pandas-dev -f qt --json echo Have pandas? - conda run -n mamba-env mamba list -n pandas-dev -f pandas --json - conda run -n mamba-env mamba run --no-capture-output -n pandas-dev pip list | grep pandas + conda list -n pandas-dev -f pandas --json + conda run --no-capture-output -n pandas-dev pip list | grep pandas set -e - time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - time mamba run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 + time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Setup environment and build pandas' # - script: | -# mamba run --no-capture-output -n pandas-dev ci/run_tests.sh +# conda run --no-capture-output -n pandas-dev ci/run_tests.sh # displayName: 'Test' # - script: | # pushd /tmp -# mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" +# conda run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" # popd # displayName: 'Build versions' From ace28206d54fdf130dde6ea3793a5a7c8ccb6785 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:17:49 +0100 Subject: [PATCH 47/75] WIP --- ci/azure/posix.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 37faf97b247fb..69ae22c36fd32 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -74,8 +74,6 @@ jobs: - script: | set -eux - ls -lh - ls -lh $(Pipeline.Workspace) # export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 @@ -90,8 +88,8 @@ jobs: fi # See https://github.com/mamba-org/mamba/issues/633 - conda create -n pandas-dev - time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" + conda create -y -n pandas-dev + time conda run --no-capture-output -n mamba-env mamba env update -y -n pandas-dev --file="${ENV_FILE}" set +e echo Have qt? From 63ddf23b10e6af859dd6370de4dcd711e55ebbea Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:24:27 +0100 Subject: [PATCH 48/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 69ae22c36fd32..79fb4d531c209 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -89,7 +89,7 @@ jobs: # See https://github.com/mamba-org/mamba/issues/633 conda create -y -n pandas-dev - time conda run --no-capture-output -n mamba-env mamba env update -y -n pandas-dev --file="${ENV_FILE}" + time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" set +e echo Have qt? From d772071ab5ff01ab61c8f6d19be9596e6bf7c966 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:36:10 +0100 Subject: [PATCH 49/75] WIP --- ci/azure/posix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 79fb4d531c209..3eb17bc784200 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -27,7 +27,8 @@ jobs: # Workaround for mamba-org/mamba#488 rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) - conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' + time conda create -y -n mamba-env + time conda install -y -n mamba-env -c conda-forge 'mamba>=0.22' displayName: Install Mamba - task: Cache@2 From eb2e285ff75e2645701dc71dd22ade77849c534b Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:38:51 +0100 Subject: [PATCH 50/75] Revert "WIP" This reverts commit 7e973abeecbc0f8824bdb85c446f1afb583f00b5. --- ci/azure/posix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 3eb17bc784200..79fb4d531c209 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -27,8 +27,7 @@ jobs: # Workaround for mamba-org/mamba#488 rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) - time conda create -y -n mamba-env - time conda install -y -n mamba-env -c conda-forge 'mamba>=0.22' + conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' displayName: Install Mamba - task: Cache@2 From 46abd6c4e4d0230455a44d73b34c19018b3b3a21 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:47:31 +0100 Subject: [PATCH 51/75] WIP --- ci/azure/posix.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 79fb4d531c209..7b1b9a69f8da0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -56,25 +56,25 @@ jobs: - script: conda env list displayName: 'conda list' - # - task: Cache@2 - # inputs: - # key: conda-11 - # restoreKeys: | - # conda-restore-11 - # path: $(Pipeline.Workspace)/conda_pkgs - # cacheHitVar: CONDA_CACHE_RESTORED - # displayName: Conda download cache - - # - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs - # displayName: Create Conda pkgs directory - # condition: ne(variables.CONDA_CACHE_RESTORED, 'true') + - task: Cache@2 + inputs: + key: conda-11 + restoreKeys: | + conda-restore-11 + path: $(Pipeline.Workspace)/conda_pkgs + cacheHitVar: CONDA_CACHE_RESTORED + displayName: Conda download cache + + - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs + displayName: Create Conda pkgs directory + condition: ne(variables.CONDA_CACHE_RESTORED, 'true') - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' - script: | set -eux - # export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs + export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 echo ' - setuptools <60' >> ${ENV_FILE} @@ -98,8 +98,8 @@ jobs: conda list -n pandas-dev -f pandas --json conda run --no-capture-output -n pandas-dev pip list | grep pandas set -e - time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + #time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 + #time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Setup environment and build pandas' From ba105dab8cf9eeec5b1b188420bb12a05015d592 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 20:53:35 +0100 Subject: [PATCH 52/75] WIP --- ci/azure/posix.yml | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 7b1b9a69f8da0..f0f2ff86121eb 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,31 +50,14 @@ jobs: sccache -z displayName: Setup Sccache - - script: ls -lh $(Pipeline.Workspace)/conda_pkgs || true - displayName: 'List packages' - - script: conda env list displayName: 'conda list' - - task: Cache@2 - inputs: - key: conda-11 - restoreKeys: | - conda-restore-11 - path: $(Pipeline.Workspace)/conda_pkgs - cacheHitVar: CONDA_CACHE_RESTORED - displayName: Conda download cache - - - script: mkdir -p $(Pipeline.Workspace)/conda_pkgs - displayName: Create Conda pkgs directory - condition: ne(variables.CONDA_CACHE_RESTORED, 'true') - - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' displayName: 'Set conda path' - script: | set -eux - export CONDA_PKGS_DIRS=$(Pipeline.Workspace)/conda_pkgs # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 echo ' - setuptools <60' >> ${ENV_FILE} @@ -98,8 +81,8 @@ jobs: conda list -n pandas-dev -f pandas --json conda run --no-capture-output -n pandas-dev pip list | grep pandas set -e - #time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - #time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 + time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Setup environment and build pandas' @@ -120,9 +103,3 @@ jobs: # testResultsFiles: 'test-data.xml' # testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} # displayName: 'Publish test results' - - - script: | - find $(Pipeline.Workspace)/conda_pkgs -maxdepth 1 -type d -iname '*-*-*_*' | xargs rm -r - ls $(Pipeline.Workspace)/conda_pkgs - displayName: 'Reduce Conda cache size' - condition: ne(variables.CONDA_CACHE_RESTORED, 'true') From fdae0098f06d124cc180abb42c2666390bfd45a4 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:04:06 +0100 Subject: [PATCH 53/75] WIP --- ci/azure/posix.yml | 57 ++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index f0f2ff86121eb..f131e1c89b536 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,6 +22,11 @@ jobs: CONDA_PY: "310" steps: + - script: | + which conda + conda env list + displayName: 'conda list' + - script: | set -eux # Workaround for mamba-org/mamba#488 @@ -30,32 +35,6 @@ jobs: conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' displayName: Install Mamba - - task: Cache@2 - inputs: - key: sccache-0 - restoreKeys: | - sccache-restore-0 - path: $(Pipeline.Workspace)/.sccache - displayName: Sccache cache - - - script: | - set -eux - sccache_version=v0.2.15 - sccache_archive_name=sccache-$sccache_version-x86_64-apple-darwin - curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ - | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache - chmod +x /usr/local/bin/sccache - SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(Pipeline.Workspace)/.sccache sccache --start-server - sccache -s - sccache -z - displayName: Setup Sccache - - - script: conda env list - displayName: 'conda list' - - - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' - displayName: 'Set conda path' - - script: | set -eux @@ -81,10 +60,34 @@ jobs: conda list -n pandas-dev -f pandas --json conda run --no-capture-output -n pandas-dev pip list | grep pandas set -e + displayName: 'Setup Conda environment' + + - task: Cache@2 + inputs: + key: sccache-0 + restoreKeys: | + sccache-restore-0 + path: $(Pipeline.Workspace)/.sccache + displayName: Sccache cache + + - script: | + set -eux + sccache_version=v0.2.15 + sccache_archive_name=sccache-$sccache_version-x86_64-apple-darwin + curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ + | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache + chmod +x /usr/local/bin/sccache + SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(Pipeline.Workspace)/.sccache sccache --start-server + sccache -s + sccache -z + displayName: Setup Sccache + + - script: | + set -eux time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s - displayName: 'Setup environment and build pandas' + displayName: 'Build pandas' # - script: | # conda run --no-capture-output -n pandas-dev ci/run_tests.sh From d7a39a11acdf0909f810fdb3817f4e56eb906a9d Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:13:21 +0100 Subject: [PATCH 54/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index f131e1c89b536..4cc71ffce5f57 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -32,7 +32,7 @@ jobs: # Workaround for mamba-org/mamba#488 rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) - conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' + conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' pip displayName: Install Mamba - script: | From 205465374b0743396c730e5304eeedea390d5bc9 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:28:41 +0100 Subject: [PATCH 55/75] WIP --- ci/azure/posix.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 4cc71ffce5f57..9d90bb7a89a7c 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -32,7 +32,7 @@ jobs: # Workaround for mamba-org/mamba#488 rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) - conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' pip + conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' displayName: Install Mamba - script: | @@ -84,6 +84,9 @@ jobs: - script: | set -eux + conda run --no-capture-output -n mamba-dev mamba install -n pandas-dev pip -y + conda run --no-capture-output -n pandas-dev which python + conda run --no-capture-output -n pandas-dev python --version time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s From 6837c5711771252a023894884c0e7e3b907b4f7c Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:39:24 +0100 Subject: [PATCH 56/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9d90bb7a89a7c..484f77f6006d0 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -84,7 +84,7 @@ jobs: - script: | set -eux - conda run --no-capture-output -n mamba-dev mamba install -n pandas-dev pip -y + conda run --no-capture-output -n mamba-env mamba install -n pandas-dev pip -y conda run --no-capture-output -n pandas-dev which python conda run --no-capture-output -n pandas-dev python --version time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 From ea8cdf764a495d70ee305e529fa633ba0e968b8d Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:41:58 +0100 Subject: [PATCH 57/75] empty From e30edc86fb79e5e27e2324de1b756e6e0c13c45f Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:53:44 +0100 Subject: [PATCH 58/75] empty From fa8de16d7a3fe672ca4df037fea9b94a3214c275 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:53:46 +0100 Subject: [PATCH 59/75] WIP --- ci/azure/posix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 484f77f6006d0..7c1be3bfc1659 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -84,9 +84,8 @@ jobs: - script: | set -eux - conda run --no-capture-output -n mamba-env mamba install -n pandas-dev pip -y - conda run --no-capture-output -n pandas-dev which python conda run --no-capture-output -n pandas-dev python --version + conda run --no-capture-output -n pandas-dev python3 --version time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s From a16cf34729162220facf246e5a4e8dbfadd3fc0c Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:54:24 +0100 Subject: [PATCH 60/75] empty From d06e3cb252f167e6829ba623af14d807c0222e8e Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:56:36 +0100 Subject: [PATCH 61/75] empty From 697e889621af8971a40b4c32af424f9e18d9fc72 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 21:56:37 +0100 Subject: [PATCH 62/75] WIP --- ci/azure/posix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 7c1be3bfc1659..c688b9071f356 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -86,8 +86,8 @@ jobs: set -eux conda run --no-capture-output -n pandas-dev python --version conda run --no-capture-output -n pandas-dev python3 --version - time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python3 setup.py build_ext -v -j3 + time conda run --no-capture-output -n pandas-dev python3 -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Build pandas' From 4b952282c2741c430b38b2b9a1d7d4319b76c4be Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 11 Mar 2022 22:04:16 +0100 Subject: [PATCH 63/75] WIP --- ci/azure/posix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index c688b9071f356..3f5a4dcde2a29 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -33,6 +33,7 @@ jobs: rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' + conda activate mamba-env displayName: Install Mamba - script: | From 7dd6ae2dea9548e1edbd6551cd6cf38cf59daee8 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 16:16:26 +0100 Subject: [PATCH 64/75] WIP --- ci/azure/posix.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 3f5a4dcde2a29..e635f61d904d5 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -33,6 +33,11 @@ jobs: rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' + conda init bash + displayName: Install Mamba + + - script: | + set -eux conda activate mamba-env displayName: Install Mamba From 4a2985d7b1bcca913bff0a39cb65b62cc65cd5b4 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 16:24:40 +0100 Subject: [PATCH 65/75] WIP --- ci/azure/posix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index e635f61d904d5..533245f6133f9 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -38,6 +38,8 @@ jobs: - script: | set -eux + cat ~/.bash_profile + source ~/.bash_profile conda activate mamba-env displayName: Install Mamba From 002f5fbad4e24550a9ba22e38bfec3d55f14bcd8 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 16:30:12 +0100 Subject: [PATCH 66/75] WIP --- ci/azure/posix.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 533245f6133f9..8a604030981e5 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -36,13 +36,6 @@ jobs: conda init bash displayName: Install Mamba - - script: | - set -eux - cat ~/.bash_profile - source ~/.bash_profile - conda activate mamba-env - displayName: Install Mamba - - script: | set -eux @@ -61,12 +54,13 @@ jobs: conda create -y -n pandas-dev time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" + source ~/.bash_profile && conda activate pandas-dev set +e echo Have qt? - conda list -n pandas-dev -f qt --json + conda list -f qt --json echo Have pandas? - conda list -n pandas-dev -f pandas --json - conda run --no-capture-output -n pandas-dev pip list | grep pandas + conda list -f pandas --json + conda run --no-capture-output pip list | grep pandas set -e displayName: 'Setup Conda environment' @@ -92,10 +86,13 @@ jobs: - script: | set -eux - conda run --no-capture-output -n pandas-dev python --version - conda run --no-capture-output -n pandas-dev python3 --version - time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python3 setup.py build_ext -v -j3 - time conda run --no-capture-output -n pandas-dev python3 -m pip install --force-reinstall --no-build-isolation -e . + source ~/.bash_profile && conda activate pandas-dev + which python + which python3 + python --version + python3 --version + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -v -j3 + time python -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Build pandas' From cff8e4253a0b19beaf65f78f9f16f57f6f01ad53 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 18:52:14 +0100 Subject: [PATCH 67/75] WIP --- ci/azure/posix.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 8a604030981e5..f65f957f695ce 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -29,12 +29,14 @@ jobs: - script: | set -eux + # Set Conda path + echo '##vso[task.prependpath]/usr/local/miniconda/condabin' # Workaround for mamba-org/mamba#488 rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' conda init bash - displayName: Install Mamba + displayName: Setup Conda and Mamba - script: | set -eux @@ -54,13 +56,12 @@ jobs: conda create -y -n pandas-dev time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" - source ~/.bash_profile && conda activate pandas-dev set +e echo Have qt? - conda list -f qt --json + conda list -n pandas-dev -f qt --json echo Have pandas? - conda list -f pandas --json - conda run --no-capture-output pip list | grep pandas + conda list -n pandas-dev -f pandas --json + conda run --no-capture-output -n pandas-dev pip list | grep pandas set -e displayName: 'Setup Conda environment' @@ -86,13 +87,8 @@ jobs: - script: | set -eux - source ~/.bash_profile && conda activate pandas-dev - which python - which python3 - python --version - python3 --version - time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -v -j3 - time python -m pip install --force-reinstall --no-build-isolation -e . + time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 + time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Build pandas' From dce2cffd588047e69e8a8a611e84a23c974b37ba Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 18:55:56 +0100 Subject: [PATCH 68/75] WIP --- ci/azure/posix.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index f65f957f695ce..4dac7e3b8cfe7 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -35,7 +35,6 @@ jobs: rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' - conda init bash displayName: Setup Conda and Mamba - script: | From ca5ace8cb891c4d1a08cbc0d87146b8c0af7ffc8 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 19:05:41 +0100 Subject: [PATCH 69/75] WIP --- ci/azure/posix.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 4dac7e3b8cfe7..9a904b371ab4b 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -31,10 +31,17 @@ jobs: set -eux # Set Conda path echo '##vso[task.prependpath]/usr/local/miniconda/condabin' + ls /usr/local/miniconda/condabin # Workaround for mamba-org/mamba#488 rm -f /usr/local/miniconda/pkgs/cache/*.json # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' + echo 1 + ls /usr/local/miniconda/condabin + echo 2 + ls /usr/local/miniconda/bin + echo 3 + ls /usr/local/miniconda/envs/mamba-env/bin displayName: Setup Conda and Mamba - script: | From 1f6515b8767dc63b2ed04de645580ab4532e99ca Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 19:17:36 +0100 Subject: [PATCH 70/75] WIP --- ci/azure/posix.yml | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9a904b371ab4b..b3b1b2286faee 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -22,27 +22,12 @@ jobs: CONDA_PY: "310" steps: - - script: | - which conda - conda env list - displayName: 'conda list' - - script: | set -eux - # Set Conda path - echo '##vso[task.prependpath]/usr/local/miniconda/condabin' - ls /usr/local/miniconda/condabin - # Workaround for mamba-org/mamba#488 - rm -f /usr/local/miniconda/pkgs/cache/*.json - # Installing Mamba to a fresh env is much faster (5 min vs. 1 min) - conda create -y -n mamba-env -c conda-forge 'mamba>=0.22' - echo 1 - ls /usr/local/miniconda/condabin - echo 2 - ls /usr/local/miniconda/bin - echo 3 - ls /usr/local/miniconda/envs/mamba-env/bin - displayName: Setup Conda and Mamba + micromamba_url="$(bash -c 'x=$(uname)-$(uname -m) x=${x,,} x=${x//darwin/osx} x=${x//_x86/}; echo https://micromamba.snakepit.net/api/micromamba/$x/latest')" + curl -sL "$micromamba_url" | bunzip2 | tar xOf - bin/micromamba > /usr/local/bin/micromamba + chmod +x /usr/local/bin/micromamba + displayName: Install Micromamba - script: | set -eux @@ -59,15 +44,14 @@ jobs: fi # See https://github.com/mamba-org/mamba/issues/633 - conda create -y -n pandas-dev - time conda run --no-capture-output -n mamba-env mamba env update -n pandas-dev --file="${ENV_FILE}" + time micromamba create -f "$ENV_FILE" set +e echo Have qt? - conda list -n pandas-dev -f qt --json + micromamba list -n pandas-dev -f qt --json echo Have pandas? - conda list -n pandas-dev -f pandas --json - conda run --no-capture-output -n pandas-dev pip list | grep pandas + micromamba list -n pandas-dev -f pandas --json + micromamba run -a '' -n pandas-dev pip list | grep pandas set -e displayName: 'Setup Conda environment' @@ -93,8 +77,8 @@ jobs: - script: | set -eux - time DISTUTILS_C_COMPILER_LAUNCHER=sccache conda run --no-capture-output -n pandas-dev python setup.py build_ext -v -j3 - time conda run --no-capture-output -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + time DISTUTILS_C_COMPILER_LAUNCHER=sccache micromamba run -a '' -n pandas-dev python setup.py build_ext -v -j3 + time micromamba run -a '' -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . sccache -s displayName: 'Build pandas' From 44fb96eb38aecc4700ab1d70f18c60b4f89ef0cb Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 19:23:32 +0100 Subject: [PATCH 71/75] WIP --- ci/azure/posix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index b3b1b2286faee..864ea9aade5fb 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -24,8 +24,7 @@ jobs: steps: - script: | set -eux - micromamba_url="$(bash -c 'x=$(uname)-$(uname -m) x=${x,,} x=${x//darwin/osx} x=${x//_x86/}; echo https://micromamba.snakepit.net/api/micromamba/$x/latest')" - curl -sL "$micromamba_url" | bunzip2 | tar xOf - bin/micromamba > /usr/local/bin/micromamba + curl -sL https://micromamba.snakepit.net/api/micromamba/osx-64/latest | bunzip2 | tar xOf - bin/micromamba > /usr/local/bin/micromamba chmod +x /usr/local/bin/micromamba displayName: Install Micromamba From 1e8ece3ad4e7e51783c4289726240d09aed4d36f Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 19:35:26 +0100 Subject: [PATCH 72/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 864ea9aade5fb..79277a9335f33 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -43,7 +43,7 @@ jobs: fi # See https://github.com/mamba-org/mamba/issues/633 - time micromamba create -f "$ENV_FILE" + time micromamba create -y -f "$ENV_FILE" set +e echo Have qt? From 45d2f1b71092ecbb0729c0730d46295d3f946474 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 19:43:53 +0100 Subject: [PATCH 73/75] WIP --- ci/azure/posix.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 79277a9335f33..9130d7d936ea1 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -76,8 +76,10 @@ jobs: - script: | set -eux + time micromamba run -a '' -n pandas-dev which python + time micromamba run -a '' -n pandas-dev python --version time DISTUTILS_C_COMPILER_LAUNCHER=sccache micromamba run -a '' -n pandas-dev python setup.py build_ext -v -j3 - time micromamba run -a '' -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -e . + time micromamba run -a '' -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -v -e . sccache -s displayName: 'Build pandas' From cd704bc9e39238a7a6775d040f4b0c15177c7e66 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 19:53:18 +0100 Subject: [PATCH 74/75] WIP --- ci/azure/posix.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index 9130d7d936ea1..a646dfcdc5483 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -50,7 +50,7 @@ jobs: micromamba list -n pandas-dev -f qt --json echo Have pandas? micromamba list -n pandas-dev -f pandas --json - micromamba run -a '' -n pandas-dev pip list | grep pandas + micromamba run -n pandas-dev pip list | grep pandas set -e displayName: 'Setup Conda environment' @@ -76,10 +76,10 @@ jobs: - script: | set -eux - time micromamba run -a '' -n pandas-dev which python - time micromamba run -a '' -n pandas-dev python --version - time DISTUTILS_C_COMPILER_LAUNCHER=sccache micromamba run -a '' -n pandas-dev python setup.py build_ext -v -j3 - time micromamba run -a '' -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -v -e . + time micromamba run -n pandas-dev which python + time micromamba run -n pandas-dev python --version + time DISTUTILS_C_COMPILER_LAUNCHER=sccache micromamba run -n pandas-dev python setup.py build_ext -v -j3 + time micromamba run -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -v -e . sccache -s displayName: 'Build pandas' From 2bdf79d333060856db0090be51d4d2c65d97670f Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 12 Mar 2022 20:03:47 +0100 Subject: [PATCH 75/75] WIP --- ci/azure/posix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index a646dfcdc5483..4d11efa597ce9 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -79,7 +79,7 @@ jobs: time micromamba run -n pandas-dev which python time micromamba run -n pandas-dev python --version time DISTUTILS_C_COMPILER_LAUNCHER=sccache micromamba run -n pandas-dev python setup.py build_ext -v -j3 - time micromamba run -n pandas-dev python -m pip install --force-reinstall --no-build-isolation -v -e . + time micromamba run -n pandas-dev python -m pip install --no-build-isolation -v -e . sccache -s displayName: 'Build pandas'