ci.yml

name: CI

on:
  push:
    branches: master
  pull_request:
    branches: master

env:
  ENV_FILE: environment.yml

jobs:
  checks:
    name: Checks
    runs-on: ubuntu-latest
    steps:

    - name: Checkout
      uses: actions/checkout@v1

    - name: Setting conda path
      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"

    - name: Looking for unwanted patterns
      run: ci/code_checks.sh patterns
      if: true

    - name: Setup environment and build pandas
      run: ci/setup_env.sh
      if: true

    - name: Linting
      run: |
        source activate pandas-dev
        ci/code_checks.sh lint
      if: true

    - name: Dependencies consistency
      run: |
        source activate pandas-dev
        ci/code_checks.sh dependencies
      if: true

    - name: Checks on imported code
      run: |
        source activate pandas-dev
        ci/code_checks.sh code
      if: true

    - name: Running doctests
      run: |
        source activate pandas-dev
        ci/code_checks.sh doctests
      if: true

    - name: Docstring validation
      run: |
        source activate pandas-dev
        ci/code_checks.sh docstrings
      if: true

    - name: Typing validation
      run: |
        source activate pandas-dev
        ci/code_checks.sh typing
      if: true

    - name: Testing docstring validation script
      run: |
        source activate pandas-dev
        pytest --capture=no --strict scripts
      if: true

    - name: Running benchmarks
      run: |
        source activate pandas-dev
        cd asv_bench
        asv check -E existing
        git remote add upstream https://github.com/pandas-dev/pandas.git
        git fetch upstream
        if git diff upstream/master --name-only | grep -q "^asv_bench/"; then
            asv machine --yes
            asv dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
            if grep "failed" benchmarks.log > /dev/null ; then
                exit 1
            fi
        else
            echo "Benchmarks did not run, no changes detected"
        fi
      if: true

    - name: Publish benchmarks artifact
      uses: actions/upload-artifact@master
      with:
        name: Benchmarks log
        path: asv_bench/benchmarks.log
      if: failure()

    - name: Create build output artifact
      run: |
        echo $STEPS_CONTEXT > build_output.json
      env:
        STEPS_CONTEXT: ${{ toJson(steps) }}
      if: job.status == 'failure'

    - name: Upload build output artifact
      uses: actions/upload-artifact@master
      with:
        name: build_output_checks
        path: build_output.json
      if: job.status == 'failure'

  web_and_docs:
    name: Web and docs
    runs-on: ubuntu-latest
    steps:

    - name: Setting conda path
      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"

    - name: Checkout
      uses: actions/checkout@v1

    - name: Setup environment and build pandas
      run: ci/setup_env.sh

    - name: Build website
      run: |
        source activate pandas-dev
        python web/pandas_web.py web/pandas --target-path=web/build

    - name: Build documentation
      run: |
        source activate pandas-dev
        doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]}

    # This can be removed when the ipython directive fails when there are errors,
    # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
    - name: Check ipython directive errors
      run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"

    - name: Merge website and docs
      run: |
        mkdir -p pandas_web/docs
        cp -r web/build/* pandas_web/
        cp -r doc/build/html/* pandas_web/docs/

    - name: Create artifact
      run: |
        tar -czvf pandas_web.tar.gz pandas_web

    - name: Upload artifact
      uses: actions/upload-artifact@master
      with:
        name: pandas_web
        path: pandas_web.tar.gz

    - name: Install rclone
      run: sudo apt install rclone -y

    - name: Set up Rclone
      run: |
        RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf
        mkdir -p `dirname $RCLONE_CONFIG_PATH`
        echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH
        echo "type = swift" >> $RCLONE_CONFIG_PATH
        echo "env_auth = false" >> $RCLONE_CONFIG_PATH
        echo "auth_version = 3" >> $RCLONE_CONFIG_PATH
        echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH
        echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH
        echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH
        echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH
        echo "domain = default" >> $RCLONE_CONFIG_PATH
        echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH
        echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH
        echo "region = BHS" >> $RCLONE_CONFIG_PATH

    - name: Sync web
      run: |
        REMOTE_PATH="dev"
        if [[ "${{ github.event_name }}" == "pr" ]]; then
            REMOTE_PATH="$REMOTE_PATH/pr/$PR_NUMBER"
        fi
        rclone sync pandas_web ovh_cloud_pandas_web:$REMOTE_PATH
      env:
        PR_NUMBER: ${{ github.event.pull_request.number }}

    - name: Create build output artifact
      run: |
        echo $STEPS_CONTEXT > build_output.json
      env:
        STEPS_CONTEXT: ${{ toJson(steps) }}
      if: job.status == 'failure'

    - name: Upload build output artifact
      uses: actions/upload-artifact@master
      with:
        name: build_output_web_and_docs
        path: build_output.json
      if: job.status == 'failure'

  tests:
    name: Tests
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        name: [py37, py37-locale, py36-minimum-versions, py36-macos, py36-32bits, py36-locale-slow, py36-np15-windows, py37-np141-windows, py37-numpydev, py36-locale-slow-old-np]
        include:

        # regular tests

        - name: py37
          os: ubuntu-latest
          env_file: travis-37.yaml
          pattern: "not slow and not network"

        - name: py37-locale
          os: ubuntu-latest
          env_file: azure-37-locale.yaml
          pattern: "not slow and not network"
          locale_override: zh_CN.UTF-8

        - name: py36-minimum-versions
          os: ubuntu-latest
          env_file: azure-36-minimum_versions.yaml
          pattern: "not slow and not network"

        - name: py36-macos
          os: macos-latest
          env_file: azure-macos-36.yaml
          pattern: "not slow and not network"

        - name: py36-32bits
          os: ubuntu-latest
          env_file: azure-36-32bit.yaml
          pattern: "not slow and not network"
          bits32: "yes"

        - name: py36-locale-slow  # TODO: this is not slow, see pattern
          os: ubuntu-latest
          env_file: azure-36-locale_slow.yaml
          pattern: "not slow and not network"
          locale_override: it_IT.UTF-8

        - name: py36-np15-windows
          os: windows-latest
          env_file: azure-windows-36.yaml
          pattern: "not slow and not network"

        - name: py37-np141-windows
          os: windows-latest
          env_file: azure-windows-37.yaml
          pattern: "not slow and not network"

        # TODO: This is commented in azure - https://github.com/pandas-dev/pandas/issues/29432
        - name: py37-numpydev
          os: ubuntu-latest
          env_file: azure-37-numpydev.yaml
          pattern: "not slow and not network"
          pandas_testing_mode: deprecate
          # TODO env vars pending to add and use in the workflow
          # TEST_ARGS: "-W error"
          # EXTRA_APT: "xsel"

        # slow tests

        - name: py36-locale-slow-old-np
          os: ubuntu-latest
          env_file: azure-36-locale.yaml
          pattern: "slow"
          locale_override: zh_CN.UTF-8

    steps:
    - name: Checkout
      uses: actions/checkout@v1

    - name: Setting conda path
      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"

    - name: Setup environment and build pandas
      shell: bash
      run: ci/setup_env.sh
      env:
        ENV_FILE: ci/deps/${{ matrix.env_file }}
        LOCALE_OVERRIDE: ${{ matrix.locale_override }}
        BITS32: ${{ matrix.bits32 }}

    - name: Run tests
      shell: bash
      run: |
        source activate pandas-dev
        ci/run_tests.sh
      env:
        PATTERN: ${{ matrix.pattern }}
        PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }}
        COVERAGE: ${{ matrix.coverage }}
        LOCALE_OVERRIDE: ${{ matrix.locale_override }}

  db_tests:
    name: DB Tests
    runs-on: ${{ matrix.os }}

    services:
      postgres:
        image: postgres:latest
        ports:
        - 5432/tcp
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

      mysql:
        image: mysql:latest
        ports:
        - 3306/tcp
        env:
          MYSQL_ROOT_PASSWORD: password
        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5

    strategy:
      fail-fast: false
      matrix:
        name: [py36-cov, py36-locale-zh]
        include:

        - name: py36-cov
          os: ubuntu-latest
          env_file: travis-36-cov.yaml
          pattern: "(not slow and not network) or (single and db)"
          pandas_testing_mode: deprecate
          coverage: true

        - name: py36-locale-zh
          os: ubuntu-latest
          env_file: travis-36-locale.yaml
          pattern: "(not slow and not network) or (single and db)"
          locale_override: zh_CN.UTF-8

    steps:
    - name: Setting conda path
      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"

    - name: Checkout
      uses: actions/checkout@v1

    - name: Create PostgreSQL database
      run: |
        sudo apt-get install -y postgresql-client
        psql --host 127.0.0.1 \
             --port ${{ job.services.postgres.ports[5432] }} \
             --username postgres \
             --command 'CREATE DATABASE pandas_nosetest;'

    - name: Create MySQL database
      run: |
        mysql --host 127.0.0.1 \
              --port ${{ job.services.mysql.ports[3306] }} \
              --user root \
              -ppassword \
              --execute "CREATE DATABASE pandas_nosetest;"

    - name: Setup environment and build pandas
      run: |
        ci/setup_env.sh
      env:
        ENV_FILE: ci/deps/${{ matrix.env_file }}
        LOCALE_OVERRIDE: ${{ matrix.locale_override }}

    - name: Run tests
      run: |
        source activate pandas-dev
        ci/run_tests.sh
      env:
        PATTERN: ${{ matrix.pattern }}
        PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }}
        COVERAGE: ${{ matrix.coverage }}
        LOCALE_OVERRIDE: ${{ matrix.locale_override }}


  finalize:
    name: Finalize build
    runs-on: ubuntu-latest
    if: true
    # TODO run when all the other jobs finish:
    needs: [checks, web_and_docs]
    steps:

    - name: Download checks artifact
      uses: actions/download-artifact@master
      with:
        name: build_output_checks
        path: build_output_checks.json

    - name: Extract errors from checks artifact
      if: true  # TODO temporary for debugging
      run: |
        export MSG="Hi @$AUTHOR, thanks for your contribution. We found the next list of errors in your pull request:"
        # TODO get actual errors
        export MSG="$MSG\n- [Actual messages from the log are not yet extracted](https://github.com/pandas-dev/pandas/actions)"
        echo "{\"body\": \"$MSG\"}" > payload.json

    - name:
      run: |
        curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$(<payload.json)" https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments