From 24f6ae6760cddc5a43eab4532fbfaada11f8f7e2 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Mar 2017 08:30:00 -0400 Subject: [PATCH 1/4] CI: fast finish travis builds for the same PR closes #12438 --- .travis.yml | 5 ++- ci/install_travis.sh | 14 +------- ci/travis_fast_finish.py | 76 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 16 deletions(-) create mode 100755 ci/travis_fast_finish.py diff --git a/.travis.yml b/.travis.yml index 67b37f1d58931..270f8c2fc76c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -177,15 +177,14 @@ matrix: - USE_CACHE=true before_install: + - echo "Checking to see if this build is outdated" + - ci/travis_fast_finish.py || { echo "Failing outdated build to end it."; exit 1; } - echo "before_install" - source ci/travis_process_gbq_encryption.sh - - echo $VIRTUAL_ENV - export PATH="$HOME/miniconda3/bin:$PATH" - df -h - - date - pwd - uname -a - - python -V - git --version - git tag - ci/before_install_travis.sh diff --git a/ci/install_travis.sh b/ci/install_travis.sh index c940083f5ae9e..6e3b38b3fb1bf 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -1,18 +1,6 @@ #!/bin/bash -# There are 2 distinct pieces that get zipped and cached -# - The venv site-packages dir including the installed dependencies -# - The pandas build artifacts, using the build cache support via -# scripts/use_build_cache.py -# -# if the user opted in to use the cache and we're on a whitelisted fork -# - if the server doesn't hold a cached version of venv/pandas build, -# do things the slow way, and put the results on the cache server -# for the next time. -# - if the cache files are available, instal some necessaries via apt -# (no compiling needed), then directly goto script and collect 200$. -# - +# edit the locale file if needed function edit_init() { if [ -n "$LOCALE_OVERRIDE" ]; then diff --git a/ci/travis_fast_finish.py b/ci/travis_fast_finish.py new file mode 100755 index 0000000000000..fd735cefb442f --- /dev/null +++ b/ci/travis_fast_finish.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# script to cancel previous travis builds for the same PR +# originally from +# https://github.com/conda-forge/staged-recipes/pull/2257 + +try: + from future_builtins import ( + map, + filter, + ) +except ImportError: + pass + +import codecs +import contextlib +import json +import os + +try: + from urllib.request import ( + Request, + urlopen, + ) +except ImportError: + from urllib2 import ( + Request, + urlopen, + ) + + +def check_latest_pr_build(repo, pr, build_num): + # Not a PR so it is latest. + if pr is None: + return True + + headers = { + "Accept": "application/vnd.travis-ci.2+json", + } + url = "https://api.travis-ci.org/repos/{repo}/builds?event_type=pull_request" + + request = Request(url.format(repo=repo), headers=headers) + with contextlib.closing(urlopen(request)) as response: + reader = codecs.getreader("utf-8") + data = json.load(reader(response)) + + # Parse the response to get a list of build numbers for this PR. + builds = data["builds"] + pr_builds = filter(lambda b: b["pull_request_number"] == pr, builds) + pr_build_nums = sorted(map(lambda b: int(b["number"]), pr_builds)) + + print("build_num: {}".format(build_num)) + print("pr_build_nums: {}".format(','.join([str(n) for n in pr_build_nums]))) + + # Check if our build number is the latest (largest) + # out of all of the builds for this PR. + if build_num < max(pr_build_nums): + return False + else: + return True + + +def main(): + repo = os.environ["TRAVIS_REPO_SLUG"] + + pr = os.environ["TRAVIS_PULL_REQUEST"] + pr = None if pr == "false" else int(pr) + build_num = int(os.environ["TRAVIS_BUILD_NUMBER"]) + print("checking for fast_finish: {}-{}-{}".format(repo, pr, build_num)) + + return int(check_latest_pr_build(repo, pr, build_num) is False) + + +if __name__ == "__main__": + import sys + sys.exit(main()) From 0c33d9bd26dcbd5d42ccd08b9fd2d06d00d5639e Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Mar 2017 08:53:58 -0400 Subject: [PATCH 2/4] tests commit --- ci/travis_fast_finish.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/travis_fast_finish.py b/ci/travis_fast_finish.py index fd735cefb442f..c2e2a9159918b 100755 --- a/ci/travis_fast_finish.py +++ b/ci/travis_fast_finish.py @@ -66,6 +66,7 @@ def main(): pr = os.environ["TRAVIS_PULL_REQUEST"] pr = None if pr == "false" else int(pr) build_num = int(os.environ["TRAVIS_BUILD_NUMBER"]) + print("checking for fast_finish: {}-{}-{}".format(repo, pr, build_num)) return int(check_latest_pr_build(repo, pr, build_num) is False) From b6f2a62e28542868fd1ca0a84b9a0bb98c8d117e Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Mar 2017 09:22:05 -0400 Subject: [PATCH 3/4] formatting --- ci/install_travis.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 6e3b38b3fb1bf..616cc948ecf7a 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -14,15 +14,18 @@ function edit_init() fi } +echo . echo "[install_travis]" edit_init home_dir=$(pwd) -echo "[home_dir: $home_dir]" +echo . +echo "[home_dir]: $home_dir" # install miniconda MINICONDA_DIR="$HOME/miniconda3" +echo . echo "[Using clean Miniconda install]" if [ -d "$MINICONDA_DIR" ]; then @@ -37,14 +40,17 @@ else fi time bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 +echo . echo "[show conda]" which conda +echo . echo "[update conda]" conda config --set ssl_verify false || exit 1 conda config --set always_yes true --set changeps1 false || exit 1 conda update -q conda +echo . echo "[add channels]" # add the pandas channel to take priority # to add extra packages @@ -61,26 +67,28 @@ fi conda info -a || exit 1 # set the compiler cache to work +echo . if [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then echo "[Using ccache]" export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH gcc=$(which gcc) - echo "[gcc: $gcc]" + echo "[gcc]: $gcc" ccache=$(which ccache) - echo "[ccache: $ccache]" + echo "[ccache]: $ccache" export CC='ccache gcc' elif [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then echo "[Using ccache]" time brew install ccache export PATH=/usr/local/opt/ccache/libexec:$PATH gcc=$(which gcc) - echo "[gcc: $gcc]" + echo "[gcc]: $gcc" ccache=$(which ccache) - echo "[ccache: $ccache]" + echo "[ccache]: $ccache" else echo "[Not using ccache]" fi +echo . echo "[create env]" # may have installation instructions for this build @@ -94,6 +102,7 @@ else fi # build deps +echo . echo "[build installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build" if [ -e ${REQ} ]; then @@ -101,6 +110,7 @@ if [ -e ${REQ} ]; then fi # may have addtl installation instructions for this build +echo . echo "[build addtl installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build.sh" if [ -e ${REQ} ]; then @@ -120,6 +130,7 @@ if [ "$COVERAGE" ]; then pip install coverage pytest-cov fi +echo . if [ "$BUILD_TEST" ]; then # build & install testing @@ -139,6 +150,7 @@ else fi # we may have run installations +echo . echo "[conda installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run" if [ -e ${REQ} ]; then @@ -146,6 +158,7 @@ if [ -e ${REQ} ]; then fi # we may have additional pip installs +echo . echo "[pip installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip" if [ -e ${REQ} ]; then @@ -153,6 +166,7 @@ if [ -e ${REQ} ]; then fi # may have addtl installation instructions for this build +echo . echo "[addtl installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh" if [ -e ${REQ} ]; then @@ -164,14 +178,17 @@ if [ -z "$BUILD_TEST" ]; then # remove any installed pandas package # w/o removing anything else + echo . echo "[removing installed pandas]" conda remove pandas --force # install our pandas + echo . echo "[running setup.py develop]" python setup.py develop || exit 1 fi +echo . echo "[done]" exit 0 From 64f217e72c161e8d1e15f34ca288054d73124a4e Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Mar 2017 09:35:22 -0400 Subject: [PATCH 4/4] replace . by space --- ci/install_travis.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ci/install_travis.sh b/ci/install_travis.sh index 616cc948ecf7a..66633c0592748 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -14,18 +14,18 @@ function edit_init() fi } -echo . +echo echo "[install_travis]" edit_init home_dir=$(pwd) -echo . +echo echo "[home_dir]: $home_dir" # install miniconda MINICONDA_DIR="$HOME/miniconda3" -echo . +echo echo "[Using clean Miniconda install]" if [ -d "$MINICONDA_DIR" ]; then @@ -40,17 +40,17 @@ else fi time bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 -echo . +echo echo "[show conda]" which conda -echo . +echo echo "[update conda]" conda config --set ssl_verify false || exit 1 conda config --set always_yes true --set changeps1 false || exit 1 conda update -q conda -echo . +echo echo "[add channels]" # add the pandas channel to take priority # to add extra packages @@ -67,7 +67,7 @@ fi conda info -a || exit 1 # set the compiler cache to work -echo . +echo if [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then echo "[Using ccache]" export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH @@ -88,7 +88,7 @@ else echo "[Not using ccache]" fi -echo . +echo echo "[create env]" # may have installation instructions for this build @@ -102,7 +102,7 @@ else fi # build deps -echo . +echo echo "[build installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build" if [ -e ${REQ} ]; then @@ -110,7 +110,7 @@ if [ -e ${REQ} ]; then fi # may have addtl installation instructions for this build -echo . +echo echo "[build addtl installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build.sh" if [ -e ${REQ} ]; then @@ -130,7 +130,7 @@ if [ "$COVERAGE" ]; then pip install coverage pytest-cov fi -echo . +echo if [ "$BUILD_TEST" ]; then # build & install testing @@ -150,7 +150,7 @@ else fi # we may have run installations -echo . +echo echo "[conda installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run" if [ -e ${REQ} ]; then @@ -158,7 +158,7 @@ if [ -e ${REQ} ]; then fi # we may have additional pip installs -echo . +echo echo "[pip installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip" if [ -e ${REQ} ]; then @@ -166,7 +166,7 @@ if [ -e ${REQ} ]; then fi # may have addtl installation instructions for this build -echo . +echo echo "[addtl installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh" if [ -e ${REQ} ]; then @@ -178,17 +178,17 @@ if [ -z "$BUILD_TEST" ]; then # remove any installed pandas package # w/o removing anything else - echo . + echo echo "[removing installed pandas]" conda remove pandas --force # install our pandas - echo . + echo echo "[running setup.py develop]" python setup.py develop || exit 1 fi -echo . +echo echo "[done]" exit 0