WIP: All the CI in GitHub actions (DO NOT MERGE)

.github/workflows/ci.yml

@@ -95,3 +95,310 @@ jobs:
         name: Benchmarks log
         path: asv_bench/benchmarks.log
       if: failure()
+
+    - name: Create build output artifact
+      run: |
+        echo $STEPS_CONTEXT > build_output.json
+      env:
+        STEPS_CONTEXT: ${{ toJson(steps) }}
+      if: job.status == 'failure'
+
+    - name: Upload build output artifact
+      uses: actions/upload-artifact@master
+      with:
+        name: build_output_checks
+        path: build_output.json
+      if: job.status == 'failure'
+
+  web_and_docs:
+    name: Web and docs
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+
+    - name: Build website
+      run: |
+        source activate pandas-dev
+        python web/pandas_web.py web/pandas --target-path=web/build
+
+    - name: Build documentation
+      run: |
+        source activate pandas-dev
+        doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]}
+
+    # This can be removed when the ipython directive fails when there are errors,
+    # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
+    - name: Check ipython directive errors
+      run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
+
+    - name: Merge website and docs
+      run: |
+        mkdir -p pandas_web/docs
+        cp -r web/build/* pandas_web/
+        cp -r doc/build/html/* pandas_web/docs/
+
+    - name: Create artifact
+      run: |
+        tar -czvf pandas_web.tar.gz pandas_web
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@master
+      with:
+        name: pandas_web
+        path: pandas_web.tar.gz
+
+    - name: Install rclone
+      run: sudo apt install rclone -y
+
+    - name: Set up Rclone
+      run: |
+        RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf
+        mkdir -p `dirname $RCLONE_CONFIG_PATH`
+        echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH
+        echo "type = swift" >> $RCLONE_CONFIG_PATH
+        echo "env_auth = false" >> $RCLONE_CONFIG_PATH
+        echo "auth_version = 3" >> $RCLONE_CONFIG_PATH
+        echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH
+        echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH
+        echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH
+        echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH
+        echo "domain = default" >> $RCLONE_CONFIG_PATH
+        echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH
+        echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH
+        echo "region = BHS" >> $RCLONE_CONFIG_PATH
+
+    - name: Sync web
+      run: |
+        REMOTE_PATH="dev"
+        if [[ "${{ github.event_name }}" == "pr" ]]; then
+            REMOTE_PATH="$REMOTE_PATH/pr/$PR_NUMBER"
+        fi
+        rclone sync pandas_web ovh_cloud_pandas_web:$REMOTE_PATH
+      env:
+        PR_NUMBER: ${{ github.event.pull_request.number }}
+
+    - name: Create build output artifact
+      run: |
+        echo $STEPS_CONTEXT > build_output.json
+      env:
+        STEPS_CONTEXT: ${{ toJson(steps) }}
+      if: job.status == 'failure'
+
+    - name: Upload build output artifact
+      uses: actions/upload-artifact@master
+      with:
+        name: build_output_web_and_docs
+        path: build_output.json
+      if: job.status == 'failure'
+
+  tests:
+    name: Tests
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        name: [py37, py37-locale, py36-minimum-versions, py36-macos, py36-32bits, py36-locale-slow, py36-np15-windows, py37-np141-windows, py37-numpydev, py36-locale-slow-old-np]
+        include:
+
+        # regular tests
+
+        - name: py37
+          os: ubuntu-latest
+          env_file: travis-37.yaml
+          pattern: "not slow and not network"
+
+        - name: py37-locale
+          os: ubuntu-latest
+          env_file: azure-37-locale.yaml
+          pattern: "not slow and not network"
+          locale_override: zh_CN.UTF-8
+
+        - name: py36-minimum-versions
+          os: ubuntu-latest
+          env_file: azure-36-minimum_versions.yaml
+          pattern: "not slow and not network"
+
+        - name: py36-macos
+          os: macos-latest
+          env_file: azure-macos-36.yaml
+          pattern: "not slow and not network"
+
+        - name: py36-32bits
+          os: ubuntu-latest
+          env_file: azure-36-32bit.yaml
+          pattern: "not slow and not network"
+          bits32: "yes"
+
+        - name: py36-locale-slow  # TODO: this is not slow, see pattern
+          os: ubuntu-latest
+          env_file: azure-36-locale_slow.yaml
+          pattern: "not slow and not network"
+          locale_override: it_IT.UTF-8
+
+        - name: py36-np15-windows
+          os: windows-latest
+          env_file: azure-windows-36.yaml
+          pattern: "not slow and not network"
+
+        - name: py37-np141-windows
+          os: windows-latest
+          env_file: azure-windows-37.yaml
+          pattern: "not slow and not network"
+
+        # TODO: This is commented in azure - https://github.com/pandas-dev/pandas/issues/29432
+        - name: py37-numpydev
+          os: ubuntu-latest
+          env_file: azure-37-numpydev.yaml
+          pattern: "not slow and not network"
+          pandas_testing_mode: deprecate
+          # TODO env vars pending to add and use in the workflow
+          # TEST_ARGS: "-W error"
+          # EXTRA_APT: "xsel"
+
+        # slow tests
+
+        - name: py36-locale-slow-old-np
+          os: ubuntu-latest
+          env_file: azure-36-locale.yaml
+          pattern: "slow"
+          locale_override: zh_CN.UTF-8
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
+    - name: Setup environment and build pandas
+      shell: bash
+      run: ci/setup_env.sh
+      env:
+        ENV_FILE: ci/deps/${{ matrix.env_file }}
+        LOCALE_OVERRIDE: ${{ matrix.locale_override }}
+        BITS32: ${{ matrix.bits32 }}
+
+    - name: Run tests
+      shell: bash
+      run: |
+        source activate pandas-dev
+        ci/run_tests.sh
+      env:
+        PATTERN: ${{ matrix.pattern }}
+        PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }}
+        COVERAGE: ${{ matrix.coverage }}
+        LOCALE_OVERRIDE: ${{ matrix.locale_override }}
+
+  db_tests:
+    name: DB Tests
+    runs-on: ${{ matrix.os }}
+
+    services:
+      postgres:
+        image: postgres:latest
+        ports:
+        - 5432/tcp
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+      mysql:
+        image: mysql:latest
+        ports:
+        - 3306/tcp
+        env:
+          MYSQL_ROOT_PASSWORD: password
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        name: [py36-cov, py36-locale-zh]
+        include:
+
+        - name: py36-cov
+          os: ubuntu-latest
+          env_file: travis-36-cov.yaml
+          pattern: "(not slow and not network) or (single and db)"
+          pandas_testing_mode: deprecate
+          coverage: true
+
+        - name: py36-locale-zh
+          os: ubuntu-latest
+          env_file: travis-36-locale.yaml
+          pattern: "(not slow and not network) or (single and db)"
+          locale_override: zh_CN.UTF-8
+
+    steps:
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Create PostgreSQL database
+      run: |
+        sudo apt-get install -y postgresql-client
+        psql --host 127.0.0.1 \
+             --port ${{ job.services.postgres.ports[5432] }} \
+             --username postgres \
+             --command 'CREATE DATABASE pandas_nosetest;'
+
+    - name: Create MySQL database
+      run: |
+        mysql --host 127.0.0.1 \
+              --port ${{ job.services.mysql.ports[3306] }} \
+              --user root \
+              -ppassword \
+              --execute "CREATE DATABASE pandas_nosetest;"
+
+    - name: Setup environment and build pandas
+      run: |
+        ci/setup_env.sh
+      env:
+        ENV_FILE: ci/deps/${{ matrix.env_file }}
+        LOCALE_OVERRIDE: ${{ matrix.locale_override }}
+
+    - name: Run tests
+      run: |
+        source activate pandas-dev
+        ci/run_tests.sh
+      env:
+        PATTERN: ${{ matrix.pattern }}
+        PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode }}
+        COVERAGE: ${{ matrix.coverage }}
+        LOCALE_OVERRIDE: ${{ matrix.locale_override }}
+
+
+  finalize:
+    name: Finalize build
+    runs-on: ubuntu-latest
+    if: true
+    # TODO run when all the other jobs finish:
+    needs: [checks, web_and_docs]
+    steps:
+
+    - name: Download checks artifact
+      uses: actions/download-artifact@master
+      with:
+        name: build_output_checks
+        path: build_output_checks.json
+
+    - name: Extract errors from checks artifact
+      if: true  # TODO temporary for debugging
+      run: |
+        export MSG="Hi @$AUTHOR, thanks for your contribution. We found the next list of errors in your pull request:"
+        # TODO get actual errors
+        export MSG="$MSG\n- [Actual messages from the log are not yet extracted](https://github.com/pandas-dev/pandas/actions)"
+        echo "{\"body\": \"$MSG\"}" > payload.json
+
+    - name:
+      run: |
+        curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$(<payload.json)" https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments

ci/setup_env.sh

@@ -24,18 +24,21 @@ fi
 echo "Install Miniconda"
 UNAME_OS=$(uname)
 if [[ "$UNAME_OS" == 'Linux' ]]; then
-    if [[ "$BITS32" == "yes" ]]; then
-        CONDA_OS="Linux-x86"
-    else
-        CONDA_OS="Linux-x86_64"
-    fi
+    CONDA_OS="Linux"
 elif [[ "$UNAME_OS" == 'Darwin' ]]; then
-    CONDA_OS="MacOSX-x86_64"
+    CONDA_OS="MacOSX"
+elif [[ "${UNAME_OS:0:7}" == 'MINGW64' ]]; then
+    CONDA_OS="Windows"
 else
   echo "OS $UNAME_OS not supported"
   exit 1
 fi
 
+CONDA_OS="${CONDA_OS}-x86"
+if [[ "$BITS32" != "yes" ]]; then
+    CONDA_OS="${CONDA_OS}_64"
+fi
+
 wget -q "https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh" -O miniconda.sh
 chmod +x miniconda.sh
 ./miniconda.sh -b
@@ -56,29 +59,6 @@ conda update -n base conda
 echo "conda info -a"
 conda info -a
 
-echo
-echo "set the compiler cache to work"
-if [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then
-    echo "Using ccache"
-    export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
-    GCC=$(which gcc)
-    echo "gcc: $GCC"
-    CCACHE=$(which ccache)
-    echo "ccache: $CCACHE"
-    export CC='ccache gcc'
-elif [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then
-    echo "Install ccache"
-    brew install ccache > /dev/null 2>&1
-    echo "Using ccache"
-    export PATH=/usr/local/opt/ccache/libexec:$PATH
-    gcc=$(which gcc)
-    echo "gcc: $gcc"
-    CCACHE=$(which ccache)
-    echo "ccache: $CCACHE"
-else
-    echo "Not using ccache"
-fi
-
 echo "source deactivate"
 source deactivate
 
@@ -139,13 +119,4 @@ echo
 echo "conda list"
 conda list
 
-# Install DB for Linux
-if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
-  echo "installing dbs"
-  mysql -e 'create database pandas_nosetest;'
-  psql -c 'create database pandas_nosetest;' -U postgres
-else
-   echo "not using dbs on non-linux Travis builds or Azure Pipelines"
-fi
-
 echo "done"