diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index d862fa1086f97..0000000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,239 +0,0 @@ -name: CI - -on: - push: - branches: - - master - - 1.3.x - pull_request: - branches: - - master - - 1.3.x - -env: - ENV_FILE: environment.yml - PANDAS_CI: 1 - -jobs: - checks: - name: Checks - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-checks - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Install node.js (for pyright) - uses: actions/setup-node@v2 - with: - node-version: "16" - - - name: Install pyright - # note: keep version in sync with .pre-commit-config.yaml - run: npm install -g pyright@1.1.200 - - - name: Build Pandas - uses: ./.github/actions/build_pandas - - - name: Checks on imported code - run: ci/code_checks.sh code - if: always() - - - name: Running doctests - run: ci/code_checks.sh doctests - if: always() - - - name: Docstring validation - run: ci/code_checks.sh docstrings - if: always() - - - name: Typing validation - run: ci/code_checks.sh typing - if: always() - - - name: Testing docstring validation script - run: pytest scripts - if: always() - - benchmarks: - name: Benchmarks - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-benchmarks - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} - - - uses: conda-incubator/setup-miniconda@v2 - with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: strict - environment-file: ${{ env.ENV_FILE }} - use-only-tar-bz2: true - - - name: Build Pandas - uses: ./.github/actions/build_pandas - - - name: Running benchmarks - run: | - cd asv_bench - asv check -E existing - git remote add upstream https://github.com/pandas-dev/pandas.git - git fetch upstream - asv machine --yes - asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log - if grep "failed" benchmarks.log > /dev/null ; then - exit 1 - fi - if: always() - - - name: Publish benchmarks artifact - uses: actions/upload-artifact@master - with: - name: Benchmarks log - path: asv_bench/benchmarks.log - if: failure() - - web_and_docs: - name: Web and docs - runs-on: ubuntu-latest - - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up pandas - uses: ./.github/actions/setup - - - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build - - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} - - # This can be removed when the ipython directive fails when there are errors, - # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) - - name: Check ipython directive errors - run: "! grep -B10 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" - - - name: Install ssh key - run: | - mkdir -m 700 -p ~/.ssh - echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} - - - name: Copy cheatsheets into site directory - run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ - - - name: Upload web - run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} - - - name: Upload dev docs - run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} - - - name: Move docs into site directory - run: mv doc/build/html web/build/docs - - - name: Save website as an artifact - uses: actions/upload-artifact@v2 - with: - name: website - path: web/build - retention-days: 14 - - data_manager: - name: Test experimental data manager - runs-on: ubuntu-latest - services: - moto: - image: motoserver/moto - env: - AWS_ACCESS_KEY_ID: foobar_key - AWS_SECRET_ACCESS_KEY: foobar_secret - ports: - - 5000:5000 - strategy: - matrix: - pattern: ["not slow and not network and not clipboard", "slow"] - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }} - cancel-in-progress: true - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up pandas - uses: ./.github/actions/setup - - - name: Run tests - env: - PANDAS_DATA_MANAGER: array - PATTERN: ${{ matrix.pattern }} - PYTEST_WORKERS: "auto" - PYTEST_TARGET: pandas - run: | - source activate pandas-dev - ci/run_tests.sh - - - name: Print skipped tests - run: python ci/print_skipped.py diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml new file mode 100644 index 0000000000000..3a84a75be838f --- /dev/null +++ b/.github/workflows/code-checks.yml @@ -0,0 +1,158 @@ +name: Code Checks + +on: + push: + branches: + - master + - 1.3.x + pull_request: + branches: + - master + - 1.3.x + +env: + ENV_FILE: environment.yml + PANDAS_CI: 1 + +jobs: + pre_commit: + name: pre-commit + runs-on: ubuntu-latest + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.9.7' + + - name: Run pre-commit + uses: pre-commit/action@v2.0.3 + + typing_and_docstring_validation: + name: Docstring and typing validation + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-code-checks + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v2 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + + - uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "*" + channels: conda-forge + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Install node.js (for pyright) + uses: actions/setup-node@v2 + with: + node-version: "16" + + - name: Install pyright + # note: keep version in sync with .pre-commit-config.yaml + run: npm install -g pyright@1.1.200 + + - name: Build Pandas + id: build + uses: ./.github/actions/build_pandas + + - name: Run checks on imported code + run: ci/code_checks.sh code + if: ${{ steps.build.outcome == 'success' }} + + - name: Run doctests + run: ci/code_checks.sh doctests + if: ${{ steps.build.outcome == 'success' }} + + - name: Run docstring validation + run: ci/code_checks.sh docstrings + if: ${{ steps.build.outcome == 'success' }} + + - name: Run typing validation + run: ci/code_checks.sh typing + if: ${{ steps.build.outcome == 'success' }} + + - name: Run docstring validation script tests + run: pytest scripts + if: ${{ steps.build.outcome == 'success' }} + + asv-benchmarks: + name: ASV Benchmarks + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-asv-benchmarks + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v2 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }} + + - uses: conda-incubator/setup-miniconda@v2 + with: + mamba-version: "*" + channels: conda-forge + activate-environment: pandas-dev + channel-priority: strict + environment-file: ${{ env.ENV_FILE }} + use-only-tar-bz2: true + + - name: Build Pandas + id: build + uses: ./.github/actions/build_pandas + + - name: Run ASV benchmarks + run: | + cd asv_bench + asv check -E existing + git remote add upstream https://github.com/pandas-dev/pandas.git + git fetch upstream + asv machine --yes + asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log + if grep "failed" benchmarks.log > /dev/null ; then + exit 1 + fi + if: ${{ steps.build.outcome == 'success' }} + + - name: Publish benchmarks artifact + uses: actions/upload-artifact@master + with: + name: Benchmarks log + path: asv_bench/benchmarks.log + if: failure() diff --git a/.github/workflows/datamanger.yml b/.github/workflows/datamanger.yml new file mode 100644 index 0000000000000..d6c96926d0d92 --- /dev/null +++ b/.github/workflows/datamanger.yml @@ -0,0 +1,57 @@ +name: Data Manager + +on: + push: + branches: + - master + - 1.3.x + pull_request: + branches: + - master + - 1.3.x + +env: + ENV_FILE: environment.yml + PANDAS_CI: 1 + +jobs: + data_manager: + name: Test experimental data manager + runs-on: ubuntu-latest + services: + moto: + image: motoserver/moto + env: + AWS_ACCESS_KEY_ID: foobar_key + AWS_SECRET_ACCESS_KEY: foobar_secret + ports: + - 5000:5000 + strategy: + matrix: + pattern: ["not slow and not network and not clipboard", "slow"] + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }} + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up pandas + uses: ./.github/actions/setup + + - name: Run tests + env: + PANDAS_DATA_MANAGER: array + PATTERN: ${{ matrix.pattern }} + PYTEST_WORKERS: "auto" + PYTEST_TARGET: pandas + run: | + source activate pandas-dev + ci/run_tests.sh + + - name: Print skipped tests + run: python ci/print_skipped.py diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml new file mode 100644 index 0000000000000..7f830b59251d1 --- /dev/null +++ b/.github/workflows/docbuild-and-upload.yml @@ -0,0 +1,77 @@ +name: Doc Build and Upload + +on: + push: + branches: + - master + - 1.3.x + pull_request: + branches: + - master + - 1.3.x + +env: + ENV_FILE: environment.yml + PANDAS_CI: 1 + +jobs: + web_and_docs: + name: Doc Build and Upload + runs-on: ubuntu-latest + + concurrency: + # https://github.community/t/concurrecy-not-work-for-push/183068/7 + group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs + cancel-in-progress: true + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up pandas + uses: ./.github/actions/setup + + - name: Build website + run: | + source activate pandas-dev + python web/pandas_web.py web/pandas --target-path=web/build + - name: Build documentation + run: | + source activate pandas-dev + doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]} + + # This can be removed when the ipython directive fails when there are errors, + # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547) + - name: Check ipython directive errors + run: "! grep -B10 \"^<<<-------------------------------------------------------------------------$\" sphinx.log" + + - name: Install ssh key + run: | + mkdir -m 700 -p ~/.ssh + echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts + if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} + + - name: Copy cheatsheets into site directory + run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ + + - name: Upload web + run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas + if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} + + - name: Upload dev docs + run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev + if: ${{github.event_name == 'push' && github.ref == 'refs/heads/master'}} + + - name: Move docs into site directory + run: mv doc/build/html web/build/docs + + - name: Save website as an artifact + uses: actions/upload-artifact@v2 + with: + name: website + path: web/build + retention-days: 14 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 2e6c2b3d53ae7..0000000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: - - master - - 1.3.x - -jobs: - pre-commit: - runs-on: ubuntu-latest - concurrency: - # https://github.community/t/concurrecy-not-work-for-push/183068/7 - group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit - cancel-in-progress: true - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.9.7' - - uses: pre-commit/action@v2.0.0