From 43e3af0224c7ebcaca6ff963a7a297f8c3047b5e Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Thu, 24 Feb 2022 14:48:02 +0100 Subject: [PATCH 1/3] Add ccache to GHA builds - Refactor duplicated code from GHA workflows to the 'build-pandas' and 'setup' actions. - Use Mamba over Conda everywhere. - Add sccache using 'hendrikmuhs/ccache-action'. - General cleanup of GHA workflows. --- .github/actions/build-pandas/action.yml | 52 ++++++++++++++++++ .github/actions/build_pandas/action.yml | 17 ------ .github/actions/setup-sccache/action.yml | 18 +++++++ .github/actions/setup/action.yml | 64 ++++++++++++++++++++--- .github/workflows/asv-bot.yml | 21 ++------ .github/workflows/code-checks.yml | 46 ++++------------ .github/workflows/docbuild-and-upload.yml | 18 ++++--- .github/workflows/posix.yml | 46 +++------------- .github/workflows/python-dev.yml | 25 +++------ .github/workflows/sdist.yml | 21 ++++---- setup.py | 43 ++++++++++++++- 11 files changed, 221 insertions(+), 150 deletions(-) create mode 100644 .github/actions/build-pandas/action.yml delete mode 100644 .github/actions/build_pandas/action.yml create mode 100644 .github/actions/setup-sccache/action.yml diff --git a/.github/actions/build-pandas/action.yml b/.github/actions/build-pandas/action.yml new file mode 100644 index 0000000000000..e3e393c64544e --- /dev/null +++ b/.github/actions/build-pandas/action.yml @@ -0,0 +1,52 @@ +name: Build pandas +description: Rebuilds the C extensions and installs pandas +inputs: + use-login-shell: + description: "Use 'bash -l' as shell (required for Conda envs)" + default: true +runs: + using: composite + steps: + # Create a shell wrapper to be able to call "bash" or "bash -l" depending + # on the "use-login-shell" arguments. + # We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments. + - name: Set shell + shell: bash + run: | + if [ ${{ inputs.use-login-shell }} = true ]; then + args="-l" + fi + echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell + cat /tmp/_build_pandas_shell + + - name: Environment Detail + shell: bash /tmp/_build_pandas_shell {0} + run: | + if which conda; then + conda info + conda list + fi + if which pip; then + pip list + fi + python --version + + - name: Get Python version + id: get-python-version + shell: bash /tmp/_build_pandas_shell {0} + run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')" + + - name: Set up sccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ steps.get-python-version.outputs.version }} + + - name: Build Pandas + shell: bash /tmp/_build_pandas_shell {0} + run: | + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -vv -j 2 + python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index + + - name: Build Version + shell: bash /tmp/_build_pandas_shell {0} + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml deleted file mode 100644 index 2e4bfea165316..0000000000000 --- a/.github/actions/build_pandas/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Build pandas -description: Rebuilds the C extensions and installs pandas -runs: - using: composite - steps: - - - name: Environment Detail - run: | - conda info - conda list - shell: bash -l {0} - - - name: Build Pandas - run: | - python setup.py build_ext -j 2 - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index - shell: bash -l {0} diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml new file mode 100644 index 0000000000000..c954d185f72fa --- /dev/null +++ b/.github/actions/setup-sccache/action.yml @@ -0,0 +1,18 @@ +name: Setup sccache +inputs: + extra-cache-key: + required: false + default: '' +runs: + using: composite + steps: + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash + + - name: Setup sccache + uses: jonashaag/ccache-action@sccache-2 + with: + sccache: true + key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9ef00e7a85a6f..dd0a618a4dc71 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,12 +1,64 @@ name: Set up pandas description: Runs all the setup steps required to have a built pandas ready to use +inputs: + environment-file: + default: environment.yml + pyarrow-version: + required: false + is-pypy: + default: false + activate-environment: + default: pandas-dev + python-version: + required: false runs: using: composite steps: - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - shell: bash -l {0} + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash - - name: Setup environment and build pandas - run: ci/setup_env.sh - shell: bash -l {0} + - name: Cache Conda packages + uses: actions/cache@v2 + with: + path: ~/conda_pkgs_dir + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }} + + - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} + run: | + grep -q '\- pyarrow' ${{ inputs.environment-file }} + sed -i "s/- pyarrow/- pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} + cat ${{ inputs.environment-file }} + shell: bash + if: ${{ inputs.pyarrow-version }} + + - name: Setup Mambaforge and install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }}) + uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "0.21.2" + use-mamba: true + channels: conda-forge + activate-environment: ${{ inputs.activate-environment }} + channel-priority: strict + environment-file: ${{ inputs.environment-file }} + python-version: ${{ inputs.python-version }} + use-only-tar-bz2: true + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Pin setuptools (GH#44980) + run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0' + shell: bash + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Setup PyPy + uses: actions/setup-python@v2 + with: + python-version: "pypy-3.8" + if: ${{ inputs.is-pypy == 'true' }} + + - name: Setup PyPy dependencies + # TODO: re-enable cov, its slowing the tests down though + run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 + shell: bash + if: ${{ inputs.is-pypy == 'true' }} diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index f3946aeb84a63..36168bee50687 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -6,8 +6,7 @@ on: - created env: - ENV_FILE: environment.yml - COMMENT: ${{github.event.comment.body}} + COMMENT: ${{ github.event.comment.body }} jobs: autotune: @@ -33,20 +32,10 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - # Although asv sets up its own env, deps are still needed - # during discovery process - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + # Although asv sets up its own env, deps are still needed + # during discovery process + - name: Set up Conda + uses: ./.github/actions/setup - name: Run benchmarks id: bench diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 58bef05940af1..f097c4d53f99e 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -11,7 +11,6 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: @@ -52,20 +51,13 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + - name: Set up Conda + uses: ./.github/actions/setup - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Build Pandas + uses: ./.github/actions/build-pandas + id: build + continue-on-error: true - name: Install node.js (for pyright) uses: actions/setup-node@v2 @@ -76,10 +68,6 @@ jobs: # note: keep version in sync with .pre-commit-config.yaml run: npm install -g pyright@1.1.212 - - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas - - name: Run checks on imported code run: ci/code_checks.sh code if: ${{ steps.build.outcome == 'success' }} @@ -94,11 +82,9 @@ jobs: - name: Run typing validation run: ci/code_checks.sh typing - if: ${{ steps.build.outcome == 'success' }} - name: Run docstring validation script tests run: pytest scripts - if: ${{ steps.build.outcome == 'success' }} asv-benchmarks: name: ASV Benchmarks @@ -118,24 +104,11 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Set up Conda + uses: ./.github/actions/setup - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas + uses: ./.github/actions/build-pandas - name: Run ASV benchmarks run: | @@ -148,7 +121,6 @@ jobs: if grep "failed" benchmarks.log > /dev/null ; then exit 1 fi - if: ${{ steps.build.outcome == 'success' }} - name: Publish benchmarks artifact uses: actions/upload-artifact@v2 diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 4cce75779d750..1d8d171078c07 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -11,13 +11,15 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: web_and_docs: name: Doc Build and Upload runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -30,17 +32,17 @@ jobs: with: fetch-depth: 0 - - name: Set up pandas + - name: Set up Conda uses: ./.github/actions/setup + - name: Build pandas + uses: ./.github/actions/build-pandas + - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build + run: python web/pandas_web.py web/pandas --target-path=web/build + - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors + run: doc/make.py --warnings-are-errors - name: Install ssh key run: | diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index fe225add5bde7..de8b9b3c6b921 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -14,6 +14,7 @@ on: env: PANDAS_CI: 1 + PYTEST_TARGET: pandas jobs: pytest: @@ -125,57 +126,24 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} - - name: Extra installs # xsel for clipboard tests run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + uses: ./.github/actions/setup with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: flexible environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support + pyarrow-version: ${{ matrix.pyarrow_version }} + is-pypy: ${{ env.IS_PYPY }} - - name: Upgrade Arrow version - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} - if: ${{ matrix.pyarrow_version }} - - - name: Setup PyPy - uses: actions/setup-python@v2 - with: - python-version: "pypy-3.8" - if: ${{ env.IS_PYPY == 'true' }} - - - name: Setup PyPy dependencies - shell: bash - run: | - # TODO: re-enable cov, its slowing the tests down though - pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 - if: ${{ env.IS_PYPY == 'true' }} - - - name: Build Pandas - uses: ./.github/actions/build_pandas + - name: Build pandas + uses: ./.github/actions/build-pandas - name: Test run: ci/run_tests.sh # TODO: Don't continue on error for PyPy continue-on-error: ${{ env.IS_PYPY == 'true' }} - if: always() - - - name: Build Version - run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - name: Publish test results uses: actions/upload-artifact@v2 diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c287827206336..3ec5aaf9372b9 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,11 +21,11 @@ on: - "doc/**" env: - PYTEST_WORKERS: "auto" PANDAS_CI: 1 + PYTEST_WORKERS: "auto" + PYTEST_TARGET: pandas PATTERN: "not slow and not network and not clipboard and not single_cpu" COVERAGE: true - PYTEST_TARGET: pandas jobs: build: @@ -56,27 +56,19 @@ jobs: # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install dependencies - shell: bash run: | python -m pip install --upgrade pip "setuptools<60.0.0" wheel pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy pip install git+https://github.com/nedbat/coveragepy.git pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" + - name: Build pandas + uses: ./.github/actions/build-pandas + with: + use-login-shell: false - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh + run: bash ci/run_tests.sh - name: Publish test results uses: actions/upload-artifact@v2 @@ -86,8 +78,7 @@ jobs: if: failure() - name: Report Coverage - run: | - coverage report -m + run: coverage report -m - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 1d9c8701d42d4..d886d944eac6d 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -54,18 +54,24 @@ jobs: pip list python setup.py sdist --formats=gztar - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda + uses: ./.github/actions/setup with: + environment-file: "" activate-environment: pandas-sdist - channels: conda-forge - python-version: '${{ matrix.python-version }}' + python-version: ${{ matrix.python-version }} + + - name: Set up ccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ matrix.python-version }} # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install pandas from sdist run: | python -m pip install --upgrade "setuptools<60.0.0" pip list - python -m pip install dist/*.gz + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python -m pip install -vv dist/*.gz - name: Force oldest supported NumPy run: | @@ -78,8 +84,5 @@ jobs: pip install numpy==1.21.2 ;; esac - - name: Import pandas - run: | - cd .. - conda list - python -c "import pandas; pandas.show_versions();" + - name: Build Version + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/setup.py b/setup.py index 62704dc4423c8..cd8a5a49322b6 100755 --- a/setup.py +++ b/setup.py @@ -74,7 +74,48 @@ def is_platform_mac(): _pxi_dep[module] = pxi_files -class build_ext(_build_ext): +class CompilerLauncherMixin: + """Add "compiler launchers" to distutils. + + We use this to be able to run the Pandas build using "ccache". + + A compiler launcher is a program that is invoked instead of invoking the + compiler directly. It is passed the full compiler invocation command line. + + A similar feature exists in CMake, see + https://cmake.org/cmake/help/latest/prop_tgt/LANG_COMPILER_LAUNCHER.html. + """ + + __is_set_up = False + + def build_extensions(self): + # Integrate into "build_ext" + self.__setup() + super().build_extensions() + + def build_libraries(self): + # Integrate into "build_clib" + self.__setup() + super().build_extensions() + + def __setup(self): + if self.__is_set_up: + return + self.__is_set_up = True + compiler_launcher = os.getenv("DISTUTILS_C_COMPILER_LAUNCHER") + if compiler_launcher: + + def spawn_with_compiler_launcher(cmd): + exclude_programs = ("link.exe",) + if not cmd[0].endswith(exclude_programs): + cmd = [compiler_launcher] + cmd + return original_spawn(cmd) + + original_spawn = self.compiler.spawn + self.compiler.spawn = spawn_with_compiler_launcher + + +class build_ext(CompilerLauncherMixin, _build_ext): @classmethod def render_templates(cls, pxifiles): for pxifile in pxifiles: From 44881357e7646507ea0554be7a7501a43e63ed99 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 28 Feb 2022 09:47:47 +0100 Subject: [PATCH 2/3] Switch order of name: to be less confusing --- .github/workflows/posix.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index de8b9b3c6b921..ad96753b10d72 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -31,38 +31,38 @@ jobs: # even if tests are skipped/xfailed pyarrow_version: ["5", "6", "7"] include: - - env_file: actions-38-downstream_compat.yaml + - name: "Downstream Compat" + env_file: actions-38-downstream_compat.yaml pattern: "not slow and not network and not single_cpu" pytest_target: "pandas/tests/test_downstream.py" - name: "Downstream Compat" - - env_file: actions-38-minimum_versions.yaml + - name: "Minimum Versions" + env_file: actions-38-minimum_versions.yaml pattern: "not slow and not network and not single_cpu" - name: "Minimum Versions" - - env_file: actions-38.yaml + - name: "Locale: it_IT.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-it" lang: "it_IT.utf8" lc_all: "it_IT.utf8" - name: "Locale: it_IT.utf8" - - env_file: actions-38.yaml + - name: "Locale: zh_CN.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-zh-hans" lang: "zh_CN.utf8" lc_all: "zh_CN.utf8" - name: "Locale: zh_CN.utf8" - - env_file: actions-38.yaml + - name: "Data Manager" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" pandas_data_manager: "array" - name: "Data Manager" - - env_file: actions-pypy-38.yaml + - name: "Pypy" + env_file: actions-pypy-38.yaml pattern: "not slow and not network and not single_cpu" test_args: "--max-worker-restart 0" - name: "Pypy" - - env_file: actions-310-numpydev.yaml + - name: "Numpy Dev" + env_file: actions-310-numpydev.yaml pattern: "not slow and not network and not single_cpu" pandas_testing_mode: "deprecate" test_args: "-W error" - name: "Numpy Dev" fail-fast: false name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} env: From b76e708d74943da5a0063428a34091d64af66dfe Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Mon, 28 Feb 2022 09:52:10 +0100 Subject: [PATCH 3/3] Fix continue condition --- .github/workflows/code-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index f097c4d53f99e..4e53961e9784e 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -82,9 +82,11 @@ jobs: - name: Run typing validation run: ci/code_checks.sh typing + if: ${{ steps.build.outcome == 'success' }} - name: Run docstring validation script tests run: pytest scripts + if: ${{ steps.build.outcome == 'success' }} asv-benchmarks: name: ASV Benchmarks