diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0e898600ba95..c8ab0d42ad775 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,104 +98,94 @@ jobs: path: asv_bench/benchmarks.log if: failure() - web_and_docs: - name: Web and docs - runs-on: ubuntu-latest - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up pandas - uses: ./.github/actions/setup - - - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build - - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} - - # This can be removed when the ipython directive fails when there are errors, - # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - - name: Check ipython directive errors - run: "! grep -B10 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" - - - name: Install ssh key - run: | - mkdir -m 700 -p ~/.ssh - echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} - - - name: Copy cheatsheets into site directory - run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ - - - name: Upload web - run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} - - - name: Upload dev docs - run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} - - - name: Move docs into site directory - run: mv doc/build/html web/build/docs - - - name: Save website as an artifact - uses: actions/upload-artifact@v2 - with: - name: website - path: web/build - retention-days: 14 - - data_manager: - name: Test experimental data manager - runs-on: ubuntu-latest - services: - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 - strategy: - matrix: - pattern: ["not slow and not network and not clipboard", "slow"] - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }} - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up pandas - uses: ./.github/actions/setup - - - name: Run tests - env: - PANDAS_DATA_MANAGER: array - PATTERN: ${{ matrix.pattern }} - PYTEST_WORKERS: "auto" - PYTEST_TARGET: pandas - run: | - source activate pandas-dev - ci/run_tests.sh - - - name: Print skipped tests - run: python ci/print_skipped.py +# web_and_docs: +# name: Web and docs +# runs-on: ubuntu-latest +# +# concurrency: +# group: ${{ github.ref }}-web-docs +# cancel-in-progress: true +# +# steps: +# - name: Checkout +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# - name: Set up pandas +# uses: ./.github/actions/setup +# +# - name: Build website +# run: | +# source activate pandas-dev +# python web/pandas_web.py web/pandas --target-path=web/build +# - name: Build documentation +# run: | +# source activate pandas-dev +# doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} +# +# # This can be removed when the ipython directive fails when there are errors, +# # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) +# - name: Check ipython directive errors +# run: "! grep -B10 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" +# +# - name: Install ssh key +# run: | +# mkdir -m 700 -p ~/.ssh +# echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa +# chmod 600 ~/.ssh/id_rsa +# echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts +# if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} +# +# - name: Copy cheatsheets into site directory +# run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ +# +# - name: Upload web +# run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas +# if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} +# +# - name: Upload dev docs +# run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev +# if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} +# +# - name: Move docs into site directory +# run: mv doc/build/html web/build/docs +# +# - name: Save website as an artifact +# uses: actions/upload-artifact@v2 +# with: +# name: website +# path: web/build +# retention-days: 14 +# +# data_manager: +# name: Test experimental data manager +# runs-on: ubuntu-latest +# strategy: +# matrix: +# pattern: ["not slow and not network and not clipboard", "slow"] +# concurrency: +# group: ${{ github.ref }}-data_manager-${{ matrix.pattern }} +# cancel-in-progress: true +# +# steps: +# - name: Checkout +# uses: actions/checkout@v2 +# with: +# fetch-depth: 0 +# +# - name: Set up pandas +# uses: ./.github/actions/setup +# +# - name: Run tests +# env: +# PANDAS_DATA_MANAGER: array +# PATTERN: ${{ matrix.pattern }} +# PYTEST_WORKERS: "auto" +# PYTEST_TARGET: pandas +# run: | +# source activate pandas-dev +# ci/run_tests.sh +# +# - name: Print skipped tests +# run: python ci/print_skipped.py diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml deleted file mode 100644 index 3fdca951ae058..0000000000000 --- a/.github/workflows/database.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: Database - -on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - paths-ignore: - - "doc/**" - -env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - PATTERN: ((not slow and not network and not clipboard) or (single and db)) - COVERAGE: true - -jobs: - Linux_py38_IO: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - - strategy: - matrix: - ENV_FILE: [ci/deps/actions-38-db-min.yaml, ci/deps/actions-38-db.yaml] - fail-fast: false - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.ENV_FILE }} - cancel-in-progress: true - - services: - mysql: - image: mysql - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: pandas - options: >- - --health-cmd "mysqladmin ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 3306:3306 - - postgres: - image: postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: pandas - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ matrix.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ matrix.ENV_FILE }} - use-only-tar-bz2: true - - - name: Build Pandas - uses: ./.github/actions/build_pandas - - - name: Test - run: pytest -m "${{ env.PATTERN }}" -n 2 --dist=loadfile --cov=pandas --cov-report=xml pandas/tests/io - if: always() - - - name: Build Version - run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - - name: Publish test results - uses: actions/upload-artifact@master - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Print skipped tests - run: python ci/print_skipped.py - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml deleted file mode 100644 index 19b96c866923f..0000000000000 --- a/.github/workflows/posix.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Posix - -on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - paths-ignore: - - "doc/**" - -env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - -jobs: - pytest: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - strategy: - matrix: - settings: [ - [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""], - [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""], - [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""], - [actions-38-slow.yaml, "slow", "", "", "", "", ""], - [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""], - [actions-39-slow.yaml, "slow", "", "", "", "", ""], - [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"], - [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""] - ] - fail-fast: false - env: - COVERAGE: true - ENV_FILE: ci/deps/${{ matrix.settings[0] }} - PATTERN: ${{ matrix.settings[1] }} - EXTRA_APT: ${{ matrix.settings[2] }} - LANG: ${{ matrix.settings[3] }} - LC_ALL: ${{ matrix.settings[4] }} - PANDAS_TESTING_MODE: ${{ matrix.settings[5] }} - TEST_ARGS: ${{ matrix.settings[6] }} - PYTEST_TARGET: pandas - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }} - cancel-in-progress: true - services: - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v2 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('${{ env.ENV_FILE }}') }} - - - name: Extra installs - run: sudo apt-get update && sudo apt-get install -y libc6-dev-i386 ${{ env.EXTRA_APT }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-dev - channel-priority: flexible - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Build Pandas - uses: ./.github/actions/build_pandas - - - name: Test - run: ci/run_tests.sh - if: always() - - - name: Build Version - run: pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - - name: Publish test results - uses: actions/upload-artifact@master - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Print skipped tests - run: python ci/print_skipped.py - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 2e6c2b3d53ae7..0000000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: - - master - - 1.3.x - -jobs: - pre-commit: - runs-on: ubuntu-latest - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit - cancel-in-progress: true - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.9.7' - - uses: pre-commit/action@v2.0.0 diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml deleted file mode 100644 index 8b3a5a23d7a97..0000000000000 --- a/.github/workflows/python-dev.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Python Dev - -on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - paths-ignore: - - "doc/**" - -env: - PYTEST_WORKERS: "auto" - PANDAS_CI: 1 - PATTERN: "not slow and not network and not clipboard" - COVERAGE: true - PYTEST_TARGET: pandas - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - - name: actions-310-dev - timeout-minutes: 80 - - concurrency: - #https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev - cancel-in-progress: true - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python Dev Version - uses: actions/setup-python@v2 - with: - python-version: '3.10-dev' - - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip setuptools wheel - pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list - - - name: Build Pandas - run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 - - - name: Build Version - run: | - python -c "import pandas; pandas.show_versions();" - - - name: Test with pytest - shell: bash - run: | - ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@master - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Print skipped tests - run: | - python ci/print_skipped.py - - - name: Report Coverage - run: | - coverage report -m - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml deleted file mode 100644 index 5fc447a395aea..0000000000000 --- a/.github/workflows/sdist.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: sdist - -on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - paths-ignore: - - "doc/**" - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 60 - defaults: - run: - shell: bash -l {0} - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist - cancel-in-progress: true - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools wheel - - # GH 39416 - pip install numpy - - - name: Build pandas sdist - run: | - pip list - python setup.py sdist --formats=gztar - - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: pandas-sdist - channels: conda-forge - python-version: '${{ matrix.python-version }}' - - - name: Install pandas from sdist - run: | - pip list - python -m pip install dist/*.gz - - - name: Force oldest supported NumPy - run: | - case "${{matrix.python-version}}" in - 3.8) - pip install numpy==1.18.5 ;; - 3.9) - pip install numpy==1.19.3 ;; - 3.10) - pip install numpy==1.21.2 ;; - esac - - - name: Import pandas - run: | - cd .. - conda list - python -c "import pandas; pandas.show_versions();" diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 6c685d09ab55a..0000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml -trigger: - branches: - include: - - master - - 1.3.x - paths: - exclude: - - 'doc/*' - -pr: - autoCancel: true - branches: - include: - - master - - 1.3.x - -variables: - PYTEST_WORKERS: auto - PYTEST_TARGET: pandas - -jobs: -# Mac and Linux use the same template -- template: ci/azure/posix.yml - parameters: - name: macOS - vmImage: macOS-10.15 - -- template: ci/azure/windows.yml - parameters: - name: Windows - vmImage: windows-2019 - -- job: py38_32bit - pool: - vmImage: ubuntu-18.04 - - steps: - - script: | - docker pull quay.io/pypa/manylinux2014_i686 - docker run -v $(pwd):/pandas quay.io/pypa/manylinux2014_i686 \ - /bin/bash -xc "cd pandas && \ - /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \ - . ~/virtualenvs/pandas-dev/bin/activate && \ - python -m pip install --no-deps -U pip wheel setuptools && \ - pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \ - python setup.py build_ext -q -j2 && \ - python -m pip install --no-build-isolation -e . && \ - pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml" - displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.8-32 bit full Linux' diff --git a/ci/code_checks.sh b/ci/code_checks.sh index 503120e486f5a..9d35c6ed7847b 100755 --- a/ci/code_checks.sh +++ b/ci/code_checks.sh @@ -93,8 +93,10 @@ fi ### DOCSTRINGS ### if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then + touch segfault.log MSG='Validate docstrings (GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS01, SS02, SS03, SS04, SS05, PR03, PR04, PR05, PR08, PRO9, PR10, EX04, RT01, RT04, RT05, SA02, SA03)' ; echo $MSG - $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR08,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03 + python3 -X faulthandler $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS02,SS03,SS04,SS05,PR03,PR04,PR05,PR08,PR09,PR10,EX04,RT01,RT04,RT05,SA02,SA03 + cat segfault.log RET=$(($RET + $?)) ; echo $MSG "DONE" fi diff --git a/scripts/validate_docstrings.py b/scripts/validate_docstrings.py index dcb002fd975c4..e6d222e52988b 100755 --- a/scripts/validate_docstrings.py +++ b/scripts/validate_docstrings.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Analyze docstrings to detect errors. @@ -17,6 +16,7 @@ import argparse import doctest +import faulthandler import importlib import io import json @@ -36,6 +36,8 @@ import pandas +faulthandler.enable(file=open("segfault.log", "w")) + # With template backend, matplotlib plots nothing matplotlib.use("template") @@ -441,12 +443,10 @@ def main(func_name, prefix, errors, output_format, ignore_deprecated): ) args = argparser.parse_args() - sys.exit( - main( - args.function, - args.prefix, - args.errors.split(",") if args.errors else None, - args.format, - args.ignore_deprecated, - ) + main( + args.function, + args.prefix, + args.errors.split(",") if args.errors else None, + args.format, + args.ignore_deprecated, )