From b06e9dc67ef9879e569eef673f453a597d456cf5 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Fri, 18 Feb 2022 00:21:41 +0100 Subject: [PATCH 1/4] fix broken Docker image build (#45889) * add "--no-build-isolation" to pip install * add job "build_docker_dev_environment" * Trigger Build * fix wrong group name * add step "Clean up dangling images" * pin base image * add newline at end of file * always pull the latest version of the base image --- .github/workflows/code-checks.yml | 24 ++++++++++++++++++++++++ Dockerfile | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 7141b02cac376..e21acc161f7d3 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -156,3 +156,27 @@ jobs: name: Benchmarks log path: asv_bench/benchmarks.log if: failure() + + build_docker_dev_environment: + name: Build Docker Dev Environment + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-build_docker_dev_environment + cancel-in-progress: true + + steps: + - name: Clean up dangling images + run: docker image prune -f + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Build image + run: docker build --pull --no-cache --tag pandas-dev-env . diff --git a/Dockerfile b/Dockerfile index 8887e80566772..2923cd60cc53b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/condaforge/miniforge3 +FROM quay.io/condaforge/miniforge3:4.11.0-0 # if you forked pandas, you can pass in your own GitHub username to use your fork # i.e. gh_username=myname @@ -45,4 +45,4 @@ RUN . /opt/conda/etc/profile.d/conda.sh \ && cd "$pandas_home" \ && export \ && python setup.py build_ext -j 4 \ - && python -m pip install -e . + && python -m pip install --no-build-isolation -e . From aa59fc484e9e9565bb6af7653ce92e5fad2d6c1a Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Tue, 29 Mar 2022 01:10:32 +0200 Subject: [PATCH 2/4] GHA: Use bash -el (#46541) (cherry picked from commit 75e604df3d4085c1d74f4705123f27bb5785eb11) --- .github/actions/build_pandas/action.yml | 4 ++-- .github/actions/setup/action.yml | 4 ++-- .github/workflows/asv-bot.yml | 2 +- .github/workflows/code-checks.yml | 6 +++--- .github/workflows/posix.yml | 3 +-- .github/workflows/python-dev.yml | 4 ++-- .github/workflows/sdist.yml | 2 +- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 2e4bfea165316..e916d5bfde5fb 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -8,10 +8,10 @@ runs: run: | conda info conda list - shell: bash -l {0} + shell: bash -el {0} - name: Build Pandas run: | python setup.py build_ext -j 2 python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index - shell: bash -l {0} + shell: bash -el {0} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9ef00e7a85a6f..c357f149f2c7f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -5,8 +5,8 @@ runs: steps: - name: Setting conda path run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - shell: bash -l {0} + shell: bash -el {0} - name: Setup environment and build pandas run: ci/setup_env.sh - shell: bash -l {0} + shell: bash -el {0} diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index f3946aeb84a63..7b974f24fc162 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # Set concurrency to prevent abuse(full runs are ~5.5 hours !!!) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index e21acc161f7d3..4e49d13c01dbe 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -39,7 +39,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -105,7 +105,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -162,7 +162,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 0d5ef807a3392..d2e939bc0ed60 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} timeout-minutes: 120 strategy: matrix: @@ -159,7 +159,6 @@ jobs: if: ${{ env.IS_PYPY == 'true' }} - name: Setup PyPy dependencies - shell: bash run: | # TODO: re-enable cov, its slowing the tests down though pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio>=0.17 hypothesis>=5.5.3 diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c287827206336..a44f85222fb0e 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -56,7 +56,7 @@ jobs: # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install dependencies - shell: bash + shell: bash -el {0} run: | python -m pip install --upgrade pip "setuptools<60.0.0" wheel pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy @@ -74,7 +74,7 @@ jobs: python -c "import pandas; pandas.show_versions();" - name: Test with pytest - shell: bash + shell: bash -el {0} run: | ci/run_tests.sh diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index dd030f1aacc44..9d81e4f8dca20 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -18,7 +18,7 @@ jobs: timeout-minutes: 60 defaults: run: - shell: bash -l {0} + shell: bash -el {0} strategy: fail-fast: false From 26d88cacb8bc4e9eba92c36e58f35e1393e47f79 Mon Sep 17 00:00:00 2001 From: Nick Crews Date: Thu, 10 Mar 2022 19:49:11 -0700 Subject: [PATCH 3/4] BLD: Upload sdist artifact from CI (#46277) * BLD: Upload sdist artifact from CI Now it is possible to inspect the sdist that is built, to see what was included and for debugging. Inspired by needing to see what was acutally included in the sdist at https://github.com/pandas-dev/pandas/pull/46271 * BLD: fix artifact name Got this error before: Error: Artifact name is not valid: refs/pull/46277/merge-3.8-sdist.gz. Contains the following character: Forward slash / This is more precise anyways. * BLD: Drop SHA from sdist artifact name Per https://github.com/pandas-dev/pandas/pull/46277#discussion_r822232617 --- .github/workflows/sdist.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 9d81e4f8dca20..6517092934c5c 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -52,6 +52,12 @@ jobs: pip list python setup.py sdist --formats=gztar + - name: Upload sdist artifact + uses: actions/upload-artifact@v3 + with: + name: ${{matrix.python-version}}-sdist.gz + path: dist/*.gz + - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: pandas-sdist From 4355b74ec8fe2621d8f9fffa875fbdd19a4cf12e Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Tue, 29 Mar 2022 01:13:31 +0200 Subject: [PATCH 4/4] Upgrade GitHub Actions versions (#46540) (cherry picked from commit 1b3bb88008011f8ce7e3ec65fcc0c31fb838352b) --- .github/workflows/asv-bot.yml | 8 +++---- .../autoupdate-pre-commit-config.yml | 4 ++-- .github/workflows/code-checks.yml | 22 +++++++++---------- .github/workflows/comment_bot.yml | 6 ++--- .github/workflows/docbuild-and-upload.yml | 4 ++-- .github/workflows/posix.yml | 10 ++++----- .github/workflows/python-dev.yml | 6 ++--- .github/workflows/sdist.yml | 6 ++--- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index 7b974f24fc162..78c224b84d5d9 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -29,19 +29,19 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache conda - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} # Although asv sets up its own env, deps are still needed # during discovery process - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v2.1.1 with: activate-environment: pandas-dev channel-priority: strict @@ -65,7 +65,7 @@ jobs: echo 'EOF' >> $GITHUB_ENV echo "REGEX=$REGEX" >> $GITHUB_ENV - - uses: actions/github-script@v5 + - uses: actions/github-script@v6 env: BENCH_OUTPUT: ${{env.BENCH_OUTPUT}} REGEX: ${{env.REGEX}} diff --git a/.github/workflows/autoupdate-pre-commit-config.yml b/.github/workflows/autoupdate-pre-commit-config.yml index 3696cba8cf2e6..d2eac234ca361 100644 --- a/.github/workflows/autoupdate-pre-commit-config.yml +++ b/.github/workflows/autoupdate-pre-commit-config.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 - name: Cache multiple paths - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cache/pre-commit diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 4e49d13c01dbe..59fb81b167bd4 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -24,10 +24,10 @@ jobs: cancel-in-progress: true steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.9.7' @@ -48,17 +48,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache conda - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v2.1.1 with: mamba-version: "*" channels: conda-forge @@ -68,7 +68,7 @@ jobs: use-only-tar-bz2: true - name: Install node.js (for pyright) - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: "16" @@ -114,17 +114,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache conda - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v2.1.1 with: mamba-version: "*" channels: conda-forge @@ -151,7 +151,7 @@ jobs: if: ${{ steps.build.outcome == 'success' }} - name: Publish benchmarks artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Benchmarks log path: asv_bench/benchmarks.log @@ -174,7 +174,7 @@ jobs: run: docker image prune -f - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 8f610fd5781ef..3824e015e8336 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -12,18 +12,18 @@ jobs: if: startsWith(github.event.comment.body, '@github-actions pre-commit') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: r-lib/actions/pr-fetch@v2 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Cache multiple paths - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cache/pre-commit ~/.cache/pip key: pre-commit-dispatched-${{ runner.os }}-build - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.8 - name: Install-pre-commit diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 4cce75779d750..bba9f62a0eca6 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -65,7 +65,7 @@ jobs: run: mv doc/build/html web/build/docs - name: Save website as an artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: website path: web/build diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index d2e939bc0ed60..ea9df610c1dff 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -121,12 +121,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Cache conda - uses: actions/cache@v2 + uses: actions/cache@v3 env: CACHE_NUMBER: 0 with: @@ -138,7 +138,7 @@ jobs: # xsel for clipboard tests run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v2.1.1 with: mamba-version: "*" channels: conda-forge @@ -153,7 +153,7 @@ jobs: if: ${{ matrix.pyarrow_version }} - name: Setup PyPy - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: "pypy-3.8" if: ${{ env.IS_PYPY == 'true' }} @@ -177,7 +177,7 @@ jobs: run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - name: Publish test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Test results path: test-data.xml diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a44f85222fb0e..8ca4cce155e96 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -45,12 +45,12 @@ jobs: cancel-in-progress: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python Dev Version - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.11-dev' @@ -79,7 +79,7 @@ jobs: ci/run_tests.sh - name: Publish test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Test results path: test-data.xml diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 6517092934c5c..8406743889f71 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -30,12 +30,12 @@ jobs: cancel-in-progress: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} @@ -58,7 +58,7 @@ jobs: name: ${{matrix.python-version}}-sdist.gz path: dist/*.gz - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v2.1.1 with: activate-environment: pandas-sdist channels: conda-forge