diff --git a/.circleci/config.yml b/.circleci/config.yml index 612552f4eac59..0d9e3ade08846 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: PANDAS_CI: "1" steps: - checkout - - run: ci/setup_env.sh + - run: .circleci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh workflows: diff --git a/ci/setup_env.sh b/.circleci/setup_env.sh similarity index 100% rename from ci/setup_env.sh rename to .circleci/setup_env.sh diff --git a/.github/actions/build-pandas/action.yml b/.github/actions/build-pandas/action.yml new file mode 100644 index 0000000000000..b91bf456e0e60 --- /dev/null +++ b/.github/actions/build-pandas/action.yml @@ -0,0 +1,39 @@ +name: Build pandas +description: Rebuilds the C extensions and installs pandas +runs: + using: composite + steps: + - name: Environment Detail + run: | + if which micromamba; then + micromamba info + micromamba list + micromamba info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none' + fi + if which pip; then + pip list + fi + python --version + shell: bash -el {0} + + - name: Get Python version + id: get-python-version + run: python3 -c "import platform as p; print(f'::set-output name=version::{p.python_implementation()}-{p.python_version()}')" + shell: bash -el {0} + + - name: Set up sccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ steps.get-python-version.outputs.version }} + + - name: Build Pandas + run: | + which python + which pip + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python setup.py build_ext -v -j 2 + pip install -v -e . --no-build-isolation --no-use-pep517 --no-index + shell: bash -el {0} + + - name: Build Version + run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + shell: bash -el {0} diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml deleted file mode 100644 index 2e4bfea165316..0000000000000 --- a/.github/actions/build_pandas/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Build pandas -description: Rebuilds the C extensions and installs pandas -runs: - using: composite - steps: - - - name: Environment Detail - run: | - conda info - conda list - shell: bash -l {0} - - - name: Build Pandas - run: | - python setup.py build_ext -j 2 - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index - shell: bash -l {0} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 0000000000000..45c9f24118b79 --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,34 @@ +name: Run tests and report results +inputs: + check-pyarrow-version: + required: false +runs: + using: composite + steps: + - name: Check PyArrow version + run: | + # Double check that we have the expected PyArrow + pushd /tmp + python -c "import pandas; pandas.show_versions()" | egrep -i "pyarrow.+: ${{ matrix.check-pyarrow-version }}" + popd + shell: bash -el {0} + if: ${{ inputs.check-pyarrow-version }} + + - name: Test + run: ci/run_tests.sh + shell: bash -el {0} + + - name: Publish test results + uses: actions/upload-artifact@v2 + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: false + if: failure() diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml new file mode 100644 index 0000000000000..3345ff72688dd --- /dev/null +++ b/.github/actions/setup-python/action.yml @@ -0,0 +1,65 @@ +name: Setup Python and install requirements +inputs: + python-version: + required: true + architecture: + default: x64 +runs: + using: composite + steps: + # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 + - name: Create temporary requirements.txt + run: | + # Drop cache at least once per month + month_today="$(date '+%Y-%m')" + cat > requirements.txt <=6.2.5 + pytest-xdist + pytest-cov + pytest-asyncio>=0.17 + EOF + shell: bash -el {0} + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + architecture: ${{ inputs.architecture }} + cache: pip + + - name: Fix $PATH on macOS + run: | + # On macOS, the Python version we installed above is too late in $PATH + # to be effective if using "bash -l" (which we need for code that works + # with Conda envs). + cat >> ~/.bash_profile < 2**32))') + NPY_DISABLE_SVML=$bits32 pip install -r requirements.txt + pip list + shell: bash -el {0} diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml new file mode 100644 index 0000000000000..9e59cc034fe18 --- /dev/null +++ b/.github/actions/setup-sccache/action.yml @@ -0,0 +1,28 @@ +name: Setup sccache +inputs: + extra-cache-key: + required: false + default: '' +runs: + using: composite + steps: + - name: Get Date + id: get-date + run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')" + shell: bash + + - name: Fix Windows temporary directory + # On Windows, for some reason the default temporary directory provided to sccache + # may become read-only at some point. Work around by having a private tempdir. + id: mktemp + run: echo "::set-output name=tmpdir::$(cygpath -w $(mktemp -d))" + shell: bash + if: ${{ runner.os == 'Windows' }} + + - name: Setup sccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + variant: sccache + key: ${{ runner.os }}--${{ runner.arch }}--${{ github.workflow }}--${{ steps.get-date.outputs.today }}--${{ inputs.extra-cache-key }} + env: + TMP: "${{ steps.mktemp.outputs.tmpdir }}" diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9ef00e7a85a6f..668ca94ca6a7d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,12 +1,50 @@ name: Set up pandas description: Runs all the setup steps required to have a built pandas ready to use +inputs: + environment-file: + default: environment.yml + pyarrow-version: + required: false + is-pypy: + default: false + environment-name: + default: pandas-dev + python-version: + required: false runs: using: composite steps: - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - shell: bash -l {0} + - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} + run: | + grep -q ' - pyarrow' ${{ inputs.environment-file }} + sed -i"" -e "s/ - pyarrow/ - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} + cat ${{ inputs.environment-file }} + shell: bash + if: ${{ inputs.pyarrow-version }} - - name: Setup environment and build pandas - run: ci/setup_env.sh - shell: bash -l {0} + - name: Pin setuptools (GH#44980) + run: | + echo ' - setuptools <60' >> ${{ inputs.environment-file }} + shell: bash + + - name: Install ${{ inputs.environment-file }} (Python ${{ inputs.python-version }}) + uses: mamba-org/provision-with-micromamba@main + with: + environment-name: ${{ inputs.environment-name }} + environment-file: ${{ inputs.environment-file }} + extra-specs: | + ${{ inputs.python-version && format('python={0}', inputs.python-version) }} + cache-env: true + if: ${{ inputs.is-pypy == 'false' }} # No pypy3.8 support + + - name: Setup PyPy + uses: actions/setup-python@v3 + with: + python-version: "pypy-3.8" + if: ${{ inputs.is-pypy == 'true' }} + + - name: Setup PyPy dependencies + # TODO: re-enable cov, its slowing the tests down though + run: pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3 pytest-asyncio>=0.17 + shell: bash + if: ${{ inputs.is-pypy == 'true' }} diff --git a/.github/workflows/asv-bot.yml b/.github/workflows/asv-bot.yml index f3946aeb84a63..63cad09e2e223 100644 --- a/.github/workflows/asv-bot.yml +++ b/.github/workflows/asv-bot.yml @@ -6,8 +6,7 @@ on: - created env: - ENV_FILE: environment.yml - COMMENT: ${{github.event.comment.body}} + COMMENT: ${{ github.event.comment.body }} jobs: autotune: @@ -17,7 +16,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # Set concurrency to prevent abuse(full runs are ~5.5 hours !!!) @@ -29,24 +28,14 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - # Although asv sets up its own env, deps are still needed - # during discovery process - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + # Although asv sets up its own env, deps are still needed + # during discovery process + - name: Set up Conda + uses: ./.github/actions/setup - name: Run benchmarks id: bench diff --git a/.github/workflows/autoupdate-pre-commit-config.yml b/.github/workflows/autoupdate-pre-commit-config.yml index 3696cba8cf2e6..11484f98995bc 100644 --- a/.github/workflows/autoupdate-pre-commit-config.yml +++ b/.github/workflows/autoupdate-pre-commit-config.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 - name: Cache multiple paths uses: actions/cache@v2 with: diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index f32fed3b3ee68..667f3da6ce972 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -11,7 +11,6 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: @@ -24,10 +23,10 @@ jobs: cancel-in-progress: true steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.9.7' @@ -39,7 +38,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -48,24 +47,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + - name: Set up Conda + uses: ./.github/actions/setup - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Build Pandas + uses: ./.github/actions/build-pandas + id: build + continue-on-error: true - name: Install node.js (for pyright) uses: actions/setup-node@v2 @@ -76,10 +68,6 @@ jobs: # note: keep version in sync with .pre-commit-config.yaml run: npm install -g pyright@1.1.230 - - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas - - name: Run checks on imported code run: ci/code_checks.sh code if: ${{ steps.build.outcome == 'success' }} @@ -105,7 +93,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -114,28 +102,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true + - name: Set up Conda + uses: ./.github/actions/setup - name: Build Pandas - id: build - uses: ./.github/actions/build_pandas + uses: ./.github/actions/build-pandas - name: Run ASV benchmarks run: | @@ -148,7 +123,6 @@ jobs: if grep "failed" benchmarks.log > /dev/null ; then exit 1 fi - if: ${{ steps.build.outcome == 'success' }} - name: Publish benchmarks artifact uses: actions/upload-artifact@v2 @@ -162,7 +136,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -174,7 +148,7 @@ jobs: run: docker image prune -f - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml index 8f610fd5781ef..5c47a500a0f9e 100644 --- a/.github/workflows/comment_bot.yml +++ b/.github/workflows/comment_bot.yml @@ -12,7 +12,7 @@ jobs: if: startsWith(github.event.comment.body, '@github-actions pre-commit') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: r-lib/actions/pr-fetch@v2 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -23,7 +23,7 @@ jobs: ~/.cache/pre-commit ~/.cache/pip key: pre-commit-dispatched-${{ runner.os }}-build - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: python-version: 3.8 - name: Install-pre-commit diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 4cce75779d750..8b8154cc53d53 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -11,13 +11,15 @@ on: - 1.4.x env: - ENV_FILE: environment.yml PANDAS_CI: 1 jobs: web_and_docs: name: Doc Build and Upload runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 @@ -26,21 +28,21 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up pandas + - name: Set up Conda uses: ./.github/actions/setup + - name: Build pandas + uses: ./.github/actions/build-pandas + - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build + run: python web/pandas_web.py web/pandas --target-path=web/build + - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors + run: doc/make.py --warnings-are-errors - name: Install ssh key run: | diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml new file mode 100644 index 0000000000000..8d2df387d5f75 --- /dev/null +++ b/.github/workflows/macos-windows.yml @@ -0,0 +1,62 @@ +name: macOS and Windows + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + paths-ignore: + - "doc/**" + +env: + PANDAS_CI: 1 + PYTEST_TARGET: pandas + PYTEST_WORKERS: auto + PATTERN: "not slow and not db and not network and not single_cpu" + +jobs: + pytest: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -el {0} + timeout-minutes: 120 + strategy: + matrix: + os: [macos-10.15, windows-2019] + env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml] + pyarrow_version: [6] + fail-fast: false + name: ${{ matrix.name || format('{0} {1} pyarrow={2}', matrix.os, matrix.env_file, matrix.pyarrow_version) }} + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.env_file }}-${{ matrix.pyarrow_version }}-mac-win + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: wmic.exe + run: wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed + if: ${{ runner.os == 'Windows' }} + + - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + uses: ./.github/actions/setup + with: + environment-file: ci/deps/${{ matrix.env_file }} + pyarrow-version: ${{ matrix.pyarrow_version }} + + - name: Build pandas + uses: ./.github/actions/build-pandas + + - name: Run tests + uses: ./.github/actions/run-tests + with: + check-pyarrow-version: ${{ matrix.pyarrow_version }} diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index c287827206336..c9446e6cbb6c2 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -21,11 +21,11 @@ on: - "doc/**" env: - PYTEST_WORKERS: "auto" PANDAS_CI: 1 + PYTEST_TARGET: pandas + PYTEST_WORKERS: "auto" PATTERN: "not slow and not network and not clipboard and not single_cpu" COVERAGE: true - PYTEST_TARGET: pandas jobs: build: @@ -35,63 +35,43 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macOS-latest, windows-latest] + python-version: ["3.11-dev"] - name: actions-311-dev + name: ${{ matrix.python-version }} timeout-minutes: 80 + defaults: + run: + shell: bash -el {0} concurrency: #https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-dev cancel-in-progress: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python Dev Version - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} and install dependencies + uses: ./.github/actions/setup-python with: - python-version: '3.11-dev' + python-version: ${{ matrix.python-version }} - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip "setuptools<60.0.0" wheel - pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list + - name: Install NumPy nightly + run: + pip install -Ui https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 + - name: Build pandas + uses: ./.github/actions/build-pandas - - name: Build Version + - name: Install pandas from sdist run: | - python -c "import pandas; pandas.show_versions();" - - - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@v2 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Report Coverage - run: | - coverage report -m + # Double check that we have the expected Python and NumPy + python --version + pip list + python --version | grep -i "$(echo "${{ matrix.python-version }}" | sed 's/-.*//')" + pip list | egrep -i "numpy.+dev" - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true + - name: Run tests + uses: ./.github/actions/run-tests diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index 431710a49a7dd..c86901d1b8520 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -14,78 +14,83 @@ on: - "doc/**" jobs: - build: - if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}} + sdist: + #if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}} runs-on: ubuntu-latest timeout-minutes: 60 defaults: run: - shell: bash -l {0} + shell: bash -el {0} strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + # 'numpy-version' is oldest supported NumPy + include: + - python-version: "3.8" + numpy-version: "1.18.5" + - python-version: "3.9" + numpy-version: "1.19.4" + - python-version: "3.10" + numpy-version: "1.21.3" + + name: sdist Python ${{ matrix.python-version }} NumPy ${{ matrix.numpy-version }} + concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.python-version }}-sdist cancel-in-progress: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} and install dependencies + uses: ./.github/actions/setup-python with: python-version: ${{ matrix.python-version }} - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - name: Install dependencies - run: | - python -m pip install --upgrade pip "setuptools<60.0.0" wheel - - # GH 39416 - pip install numpy - - name: Build pandas sdist - run: | - pip list - python setup.py sdist --formats=gztar + run: python setup.py sdist --formats=gztar - name: Upload sdist artifact uses: actions/upload-artifact@v3 with: - name: ${{matrix.python-version}}-sdist.gz + name: ${{ matrix.python-version }}-sdist.gz path: dist/*.gz - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda env with Python ${{ matrix.python-version }}, NumPy ${{ matrix.numpy-version }} + # TODO: Change this when https://github.com/mamba-org/provision-with-micromamba/pull/45 is merged + uses: jonashaag/provision-with-micromamba@cache-failure with: - activate-environment: pandas-sdist + environment-file: false + environment-name: pandas-sdist channels: conda-forge - python-version: '${{ matrix.python-version }}' + extra-specs: | + python=${{ matrix.python-version }} + setuptools<60 + numpy=${{ matrix.numpy-version }} + cache-env: true + + - name: Set up sccache + uses: ./.github/actions/setup-sccache + with: + extra-cache-key: ${{ matrix.python-version }} - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - name: Install pandas from sdist run: | - python -m pip install --upgrade "setuptools<60.0.0" + micromamba info + # Double check that we have the expected NumPy pip list - python -m pip install dist/*.gz - - - name: Force oldest supported NumPy - run: | - case "${{matrix.python-version}}" in - 3.8) - pip install numpy==1.18.5 ;; - 3.9) - pip install numpy==1.19.3 ;; - 3.10) - pip install numpy==1.21.2 ;; - esac + pip list | egrep -i "numpy.+${{ matrix.numpy-version }}" + time DISTUTILS_C_COMPILER_LAUNCHER=sccache python -m pip install -v dist/*.gz - - name: Import pandas + - name: Build Version run: | - cd .. - conda list - python -c "import pandas; pandas.show_versions();" + pushd /tmp + python -c "import pandas; pandas.show_versions()" + # Double check that we have the expected Python and NumPy + python -c "import pandas; pandas.show_versions()" | egrep -i "python.+: ${{ matrix.python-version }}" + python -c "import pandas; pandas.show_versions()" | egrep -i "numpy.+: ${{ matrix.numpy-version }}" + popd diff --git a/.github/workflows/posix.yml b/.github/workflows/ubuntu.yml similarity index 61% rename from .github/workflows/posix.yml rename to .github/workflows/ubuntu.yml index bc8791afc69f7..ecb26f5a5a200 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/ubuntu.yml @@ -1,4 +1,4 @@ -name: Posix +name: Ubuntu on: push: @@ -12,15 +12,12 @@ on: paths-ignore: - "doc/**" -env: - PANDAS_CI: 1 - jobs: pytest: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} timeout-minutes: 120 strategy: matrix: @@ -30,41 +27,42 @@ jobs: # even if tests are skipped/xfailed pyarrow_version: ["5", "7"] include: - - env_file: actions-38-downstream_compat.yaml + - name: "Downstream Compat" + env_file: actions-38-downstream_compat.yaml pattern: "not slow and not network and not single_cpu" pytest_target: "pandas/tests/test_downstream.py" - name: "Downstream Compat" - - env_file: actions-38-minimum_versions.yaml + - name: "Minimum Versions" + env_file: actions-38-minimum_versions.yaml pattern: "not slow and not network and not single_cpu" - name: "Minimum Versions" - - env_file: actions-38.yaml + - name: "Locale: it_IT.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-it" lang: "it_IT.utf8" lc_all: "it_IT.utf8" - name: "Locale: it_IT.utf8" - - env_file: actions-38.yaml + - name: "Locale: zh_CN.utf8" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" extra_apt: "language-pack-zh-hans" lang: "zh_CN.utf8" lc_all: "zh_CN.utf8" - name: "Locale: zh_CN.utf8" - - env_file: actions-38.yaml + - name: "Data Manager" + env_file: actions-38.yaml pattern: "not slow and not network and not single_cpu" pandas_data_manager: "array" - name: "Data Manager" - - env_file: actions-pypy-38.yaml + - name: "Pypy" + env_file: actions-pypy-38.yaml pattern: "not slow and not network and not single_cpu" test_args: "--max-worker-restart 0" - name: "Pypy" - - env_file: actions-310-numpydev.yaml + - name: "Numpy Dev" + env_file: actions-310-numpydev.yaml pattern: "not slow and not network and not single_cpu" pandas_testing_mode: "deprecate" test_args: "-W error" - name: "Numpy Dev" fail-fast: false name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }} env: + PANDAS_CI: 1 ENV_FILE: ci/deps/${{ matrix.env_file }} PATTERN: ${{ matrix.pattern }} EXTRA_APT: ${{ matrix.extra_apt || '' }} @@ -80,7 +78,7 @@ jobs: COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }} concurrency: # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }} + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}-ubuntu cancel-in-progress: true services: @@ -121,72 +119,26 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} - - name: Extra installs # xsel for clipboard tests run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 xsel ${{ env.EXTRA_APT }} - - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + uses: ./.github/actions/setup with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: flexible environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - if: ${{ env.IS_PYPY == 'false' }} # No pypy3.8 support + pyarrow-version: ${{ matrix.pyarrow_version }} + is-pypy: ${{ env.IS_PYPY }} - - name: Upgrade Arrow version - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=${{ matrix.pyarrow_version }} - if: ${{ matrix.pyarrow_version }} + - name: Build pandas + uses: ./.github/actions/build-pandas - - name: Setup PyPy - uses: actions/setup-python@v2 + - name: Run tests + uses: ./.github/actions/run-tests with: - python-version: "pypy-3.8" - if: ${{ env.IS_PYPY == 'true' }} - - - name: Setup PyPy dependencies - shell: bash - run: | - # TODO: re-enable cov, its slowing the tests down though - pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio hypothesis>=5.5.3 - if: ${{ env.IS_PYPY == 'true' }} - - - name: Build Pandas - uses: ./.github/actions/build_pandas - - - name: Test - run: ci/run_tests.sh - # TODO: Don't continue on error for PyPy + check-pyarrow-version: ${{ matrix.pyarrow_version }} continue-on-error: ${{ env.IS_PYPY == 'true' }} - if: always() - - - name: Build Version - run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - - name: Publish test results - uses: actions/upload-artifact@v2 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false diff --git a/.github/workflows/windows-32bit.yml b/.github/workflows/windows-32bit.yml new file mode 100644 index 0000000000000..ec6519f9d43d9 --- /dev/null +++ b/.github/workflows/windows-32bit.yml @@ -0,0 +1,56 @@ +name: Windows 32 bit builds + +on: + push: + branches: + - main + - 1.4.x + pull_request: + branches: + - main + - 1.4.x + paths-ignore: + - "doc/**" + +env: + PANDAS_CI: 1 + PYTEST_TARGET: pandas + PYTEST_WORKERS: auto + PATTERN: "not slow and not network and not clipboard and not single_cpu" + COVERAGE: true + +jobs: + windows-32bit: + runs-on: windows-latest + timeout-minutes: 120 + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.python-version }}-windows-32bit + cancel-in-progress: true + strategy: + matrix: + python-version: [3.9] + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: wmic.exe + run: wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed + + - name: Set up Python ${{ matrix.python-version }} x86 and install dependencies + uses: ./.github/actions/setup-python + with: + python-version: ${{ matrix.python-version }} + architecture: x86 + + - name: Build pandas + uses: ./.github/actions/build-pandas + + - name: Run tests + uses: ./.github/actions/run-tests diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 86cf0c79759f5..0000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml -trigger: - branches: - include: - - main - - 1.4.x - paths: - exclude: - - 'doc/**' - -pr: - autoCancel: true - branches: - include: - - main - - 1.4.x - -variables: - PYTEST_WORKERS: auto - PYTEST_TARGET: pandas - PATTERN: "not slow and not db and not network and not single_cpu" - PANDAS_CI: 1 - -jobs: -- template: ci/azure/posix.yml - parameters: - name: macOS - vmImage: macOS-10.15 - -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: windows-2019 - -- job: py38_32bit - pool: - vmImage: ubuntu-18.04 - - steps: - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - - script: | - docker pull quay.io/pypa/manylinux2014_i686 - docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ - /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ - . ~/virtualenvs/pandas-dev/bin/activate && \ - python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio hypothesis && \ - python setup.py build_ext -q -j2 && \ - python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" - displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml deleted file mode 100644 index df1d5049be33d..0000000000000 --- a/ci/azure/posix.yml +++ /dev/null @@ -1,50 +0,0 @@ -parameters: - name: '' - vmImage: '' - -jobs: -- job: ${{ parameters.name }} - timeoutInMinutes: 90 - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - py38: - ENV_FILE: ci/deps/actions-38.yaml - CONDA_PY: "38" - - py39: - ENV_FILE: ci/deps/actions-39.yaml - CONDA_PY: "39" - - py310: - ENV_FILE: ci/deps/actions-310.yaml - CONDA_PY: "310" - - steps: - - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin' - displayName: 'Set conda path' - - - script: rm /usr/local/miniconda/pkgs/cache/*.json - displayName: 'Workaround for mamba-org/mamba#488' - - - script: ci/setup_env.sh - displayName: 'Setup environment and build pandas' - - - script: | - conda run -n pandas-dev --no-capture-output ci/run_tests.sh - displayName: 'Test' - - - script: | - pushd /tmp - conda run -n pandas-dev python -c "import pandas; pandas.show_versions()" - popd - displayName: 'Build versions' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - failTaskOnFailedTests: true - testResultsFiles: 'test-data.xml' - testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} - displayName: 'Publish test results' diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml deleted file mode 100644 index 02c6564579aa2..0000000000000 --- a/ci/azure/windows.yml +++ /dev/null @@ -1,58 +0,0 @@ -parameters: - name: '' - vmImage: '' - -jobs: -- job: ${{ parameters.name }} - timeoutInMinutes: 90 - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - matrix: - py38: - ENV_FILE: ci/deps/actions-38.yaml - CONDA_PY: "38" - - py39: - ENV_FILE: ci/deps/actions-39.yaml - CONDA_PY: "39" - - py310: - ENV_FILE: ci/deps/actions-310.yaml - CONDA_PY: "310" - - steps: - - powershell: | - Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - Write-Host "##vso[task.prependpath]$HOME/miniconda3/bin" - displayName: 'Add conda to PATH' - - bash: conda install -yv -c conda-forge -n base 'mamba>=0.21.2' - displayName: 'Install mamba' - - - bash: | - # See https://github.com/mamba-org/mamba/issues/1370 - # See https://github.com/mamba-org/mamba/issues/633 - C:\\Miniconda\\condabin\\mamba.bat create -n pandas-dev - C:\\Miniconda\\condabin\\mamba.bat env update -n pandas-dev --file ci\\deps\\actions-$(CONDA_PY).yaml - # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941 - C:\\Miniconda\\condabin\\mamba.bat install -n pandas-dev 'setuptools<60' - C:\\Miniconda\\condabin\\mamba.bat list -n pandas-dev - displayName: 'Create anaconda environment' - - bash: | - source activate pandas-dev - conda list - python setup.py build_ext -q -j 2 - python -m pip install --no-build-isolation -e . - displayName: 'Build' - - bash: | - source activate pandas-dev - wmic.exe cpu get caption, deviceid, name, numberofcores, maxclockspeed - ci/run_tests.sh - displayName: 'Test' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - failTaskOnFailedTests: true - testResultsFiles: 'test-data.xml' - testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }} - displayName: 'Publish test results' diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index a5eb8a69e19da..401be14aaca02 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -9,7 +9,7 @@ dependencies: - pytest-cov - pytest-xdist>=1.31 - hypothesis>=5.5.3 - - pytest-asyncio + - pytest-asyncio>=0.17 # pandas dependencies - python-dateutil diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 37e7ea04a348a..dac1219245e84 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -11,7 +11,7 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil - - pytest-asyncio + - pytest-asyncio>=0.17 - boto3 # required dependencies diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 40f48884f1822..01415122e6076 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -12,7 +12,7 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil - - pytest-asyncio + - pytest-asyncio>=0.17 - boto3 # required dependencies diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index abba5ddd60325..f3a967f67cbc3 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -13,7 +13,7 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil - - pytest-asyncio + - pytest-asyncio>=0.17 - boto3 # required dependencies diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 9c46eca4ab989..79cd831051c2f 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -11,7 +11,7 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil - - pytest-asyncio + - pytest-asyncio>=0.17 - boto3 # required dependencies diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 89b647372d7bc..1c681104f3196 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -11,7 +11,7 @@ dependencies: - pytest-xdist>=1.31 - hypothesis>=5.5.3 - psutil - - pytest-asyncio + - pytest-asyncio>=0.17 - boto3 # required dependencies diff --git a/ci/run_tests.sh b/ci/run_tests.sh index e6de5caf955fc..026f65d248e6d 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -30,6 +30,10 @@ if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" fi +which pytest +which python +pytest -VV || true +python -VV || true echo $PYTEST_CMD sh -c "$PYTEST_CMD" diff --git a/environment.yml b/environment.yml index 5402be3d4a00b..fdcaf1f51550e 100644 --- a/environment.yml +++ b/environment.yml @@ -69,7 +69,7 @@ dependencies: - pytest>=6.0 - pytest-cov - pytest-xdist>=1.31 - - pytest-asyncio + - pytest-asyncio>=0.17 - pytest-instafail # downstream tests diff --git a/requirements-dev.txt b/requirements-dev.txt index c671dc08a263e..cfec4bfee0a67 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -47,7 +47,7 @@ flask pytest>=6.0 pytest-cov pytest-xdist>=1.31 -pytest-asyncio +pytest-asyncio>=0.17 pytest-instafail seaborn statsmodels diff --git a/setup.py b/setup.py index 62704dc4423c8..78df5e1b9571b 100755 --- a/setup.py +++ b/setup.py @@ -74,7 +74,48 @@ def is_platform_mac(): _pxi_dep[module] = pxi_files -class build_ext(_build_ext): +class CompilerLauncherMixin: + """Add "compiler launchers" to distutils. + + We use this to be able to run the Pandas build using "ccache". + + A compiler launcher is a program that is invoked instead of invoking the + compiler directly. It is passed the full compiler invocation command line. + + A similar feature exists in CMake, see + https://cmake.org/cmake/help/latest/prop_tgt/LANG_COMPILER_LAUNCHER.html. + """ + + __is_set_up = False + + def build_extensions(self): + # Integrate into "build_ext" + self.__setup() + super().build_extensions() + + def build_libraries(self): + # Integrate into "build_clib" + self.__setup() + super().build_extensions() + + def __setup(self): + if self.__is_set_up: + return + self.__is_set_up = True + compiler_launcher = os.getenv("DISTUTILS_C_COMPILER_LAUNCHER") + if compiler_launcher: + + def spawn_with_compiler_launcher(cmd): + exclude_programs = ("link.exe",) + if not cmd[0].endswith(exclude_programs): + cmd = [compiler_launcher] + cmd + return original_spawn(cmd) + + original_spawn = self.compiler.spawn + self.compiler.spawn = spawn_with_compiler_launcher + + +class build_ext(CompilerLauncherMixin, _build_ext): @classmethod def render_templates(cls, pxifiles): for pxifile in pxifiles: @@ -334,7 +375,7 @@ def run(self): extra_compile_args.append("/Z7") extra_link_args.append("/DEBUG") else: - # PANDAS_CI=1 is set by ci/setup_env.sh + # PANDAS_CI=1 is set in CI if os.environ.get("PANDAS_CI", "0") == "1": extra_compile_args.append("-Werror") if debugging_symbols_requested: