travis deduping on prs

closes #12438

Author: Jeff Reback <[email protected]>

Closes #15780 from jreback/dedupe and squashes the following commits:

64f217e [Jeff Reback] replace . by space
b6f2a62 [Jeff Reback] formatting
0c33d9b [Jeff Reback] tests commit
24f6ae6 [Jeff Reback] CI: fast finish travis builds for the same PR

Author: jreback

.travis.yml

@@ -177,15 +177,14 @@ matrix:
         - USE_CACHE=true
 
 before_install:
+  - echo "Checking to see if this build is outdated"
+  - ci/travis_fast_finish.py || { echo "Failing outdated build to end it."; exit 1; }
   - echo "before_install"
   - source ci/travis_process_gbq_encryption.sh
-  - echo $VIRTUAL_ENV
   - export PATH="$HOME/miniconda3/bin:$PATH"
   - df -h
-  - date
   - pwd
   - uname -a
-  - python -V
   - git --version
   - git tag
   - ci/before_install_travis.sh

ci/install_travis.sh

@@ -1,18 +1,6 @@
 #!/bin/bash
 
-# There are 2 distinct pieces that get zipped and cached
-# - The venv site-packages dir including the installed dependencies
-# - The pandas build artifacts, using the build cache support via
-#   scripts/use_build_cache.py
-#
-# if the user opted in to use the cache and we're on a whitelisted fork
-# - if the server doesn't hold a cached version of venv/pandas build,
-#   do things the slow way, and put the results on the cache server
-#   for the next time.
-# -  if the cache files are available, instal some necessaries via apt
-#    (no compiling needed), then directly goto script and collect 200$.
-#
-
+# edit the locale file if needed
 function edit_init()
 {
     if [ -n "$LOCALE_OVERRIDE" ]; then
@@ -26,15 +14,18 @@ function edit_init()
     fi
 }
 
+echo
 echo "[install_travis]"
 edit_init
 
 home_dir=$(pwd)
-echo "[home_dir: $home_dir]"
+echo
+echo "[home_dir]: $home_dir"
 
 # install miniconda
 MINICONDA_DIR="$HOME/miniconda3"
 
+echo
 echo "[Using clean Miniconda install]"
 
 if [ -d "$MINICONDA_DIR" ]; then
@@ -49,14 +40,17 @@ else
 fi
 time bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1
 
+echo
 echo "[show conda]"
 which conda
 
+echo
 echo "[update conda]"
 conda config --set ssl_verify false || exit 1
 conda config --set always_yes true --set changeps1 false || exit 1
 conda update -q conda
 
+echo
 echo "[add channels]"
 # add the pandas channel to take priority
 # to add extra packages
@@ -73,26 +67,28 @@ fi
 conda info -a || exit 1
 
 # set the compiler cache to work
+echo
 if [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then
     echo "[Using ccache]"
     export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
     gcc=$(which gcc)
-    echo "[gcc: $gcc]"
+    echo "[gcc]: $gcc"
     ccache=$(which ccache)
-    echo "[ccache: $ccache]"
+    echo "[ccache]: $ccache"
     export CC='ccache gcc'
 elif [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then
     echo "[Using ccache]"
     time brew install ccache
     export PATH=/usr/local/opt/ccache/libexec:$PATH
     gcc=$(which gcc)
-    echo "[gcc: $gcc]"
+    echo "[gcc]: $gcc"
     ccache=$(which ccache)
-    echo "[ccache: $ccache]"
+    echo "[ccache]: $ccache"
 else
     echo "[Not using ccache]"
 fi
 
+echo
 echo "[create env]"
 
 # may have installation instructions for this build
@@ -106,13 +102,15 @@ else
 fi
 
 # build deps
+echo
 echo "[build installs]"
 REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build"
 if [ -e ${REQ} ]; then
     time conda install -n pandas --file=${REQ} || exit 1
 fi
 
 # may have addtl installation instructions for this build
+echo
 echo "[build addtl installs]"
 REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build.sh"
 if [ -e ${REQ} ]; then
@@ -132,6 +130,7 @@ if [ "$COVERAGE" ]; then
     pip install coverage pytest-cov
 fi
 
+echo
 if [ "$BUILD_TEST" ]; then
 
     # build & install testing
@@ -151,20 +150,23 @@ else
 fi
 
 # we may have run installations
+echo
 echo "[conda installs]"
 REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run"
 if [ -e ${REQ} ]; then
     time conda install -n pandas --file=${REQ} || exit 1
 fi
 
 # we may have additional pip installs
+echo
 echo "[pip installs]"
 REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip"
 if [ -e ${REQ} ]; then
    pip install -r $REQ
 fi
 
 # may have addtl installation instructions for this build
+echo
 echo "[addtl installs]"
 REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh"
 if [ -e ${REQ} ]; then
@@ -176,14 +178,17 @@ if [ -z "$BUILD_TEST" ]; then
 
     # remove any installed pandas package
     # w/o removing anything else
+    echo
     echo "[removing installed pandas]"
     conda remove pandas --force
 
     # install our pandas
+    echo
     echo "[running setup.py develop]"
     python setup.py develop  || exit 1
 
 fi
 
+echo
 echo "[done]"
 exit 0

ci/travis_fast_finish.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+# script to cancel previous travis builds for the same PR
+# originally from
+# https://github.com/conda-forge/staged-recipes/pull/2257
+
+try:
+    from future_builtins import (
+        map,
+        filter,
+    )
+except ImportError:
+    pass
+
+import codecs
+import contextlib
+import json
+import os
+
+try:
+    from urllib.request import (
+        Request,
+        urlopen,
+    )
+except ImportError:
+    from urllib2 import (
+        Request,
+        urlopen,
+    )
+
+
+def check_latest_pr_build(repo, pr, build_num):
+    # Not a PR so it is latest.
+    if pr is None:
+        return True
+
+    headers = {
+        "Accept": "application/vnd.travis-ci.2+json",
+    }
+    url = "https://api.travis-ci.org/repos/{repo}/builds?event_type=pull_request"
+
+    request = Request(url.format(repo=repo), headers=headers)
+    with contextlib.closing(urlopen(request)) as response:
+        reader = codecs.getreader("utf-8")
+        data = json.load(reader(response))
+
+    # Parse the response to get a list of build numbers for this PR.
+    builds = data["builds"]
+    pr_builds = filter(lambda b: b["pull_request_number"] == pr, builds)
+    pr_build_nums = sorted(map(lambda b: int(b["number"]), pr_builds))
+
+    print("build_num: {}".format(build_num))
+    print("pr_build_nums: {}".format(','.join([str(n) for n in pr_build_nums])))
+
+    # Check if our build number is the latest (largest)
+    # out of all of the builds for this PR.
+    if build_num < max(pr_build_nums):
+        return False
+    else:
+        return True
+
+
+def main():
+    repo = os.environ["TRAVIS_REPO_SLUG"]
+
+    pr = os.environ["TRAVIS_PULL_REQUEST"]
+    pr = None if pr == "false" else int(pr)
+    build_num = int(os.environ["TRAVIS_BUILD_NUMBER"])
+
+    print("checking for fast_finish: {}-{}-{}".format(repo, pr, build_num))
+
+    return int(check_latest_pr_build(repo, pr, build_num) is False)
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())