CI: simplify ci setup a bit

closes #15807

Author: jreback

.travis.yml

@@ -1,9 +1,9 @@
 sudo: false
 language: python
 
-# To turn off cached miniconda, cython files and compiler cache comment out the
-# USE_CACHE=true line for the build in the matrix below. To delete caches go to
-# https://travis-ci.org/OWNER/REPOSITORY/caches or run
+# To turn off cached cython files and compiler cache
+# set NOCACHE-true
+# To delete caches go to https://travis-ci.org/OWNER/REPOSITORY/caches or run
 # travis cache --delete inside the project directory from the travis command line client
 # The cash directories will be deleted if anything in ci/ changes in a commit
 cache:
@@ -33,31 +33,31 @@ matrix:
           - $HOME/.cache # cython cache
           - $HOME/.ccache # compiler cache
       env:
-        - PYTHON_VERSION=3.5 JOB_NAME="35_osx" TEST_ARGS="--skip-slow --skip-network" JOB_TAG="_OSX" TRAVIS_PYTHON_VERSION=3.5 USE_CACHE=true
+        - JOB="3.5_OSX" TEST_ARGS="--skip-slow --skip-network" TRAVIS_PYTHON_VERSION=3.5
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7 JOB_NAME="27_slow_nnet_LOCALE" TEST_ARGS="--only-slow --skip-network" LOCALE_OVERRIDE="zh_CN.UTF-8" JOB_TAG="_LOCALE" USE_CACHE=true
+        - JOB="2.7_LOCALE" TEST_ARGS="--only-slow --skip-network" LOCALE_OVERRIDE="zh_CN.UTF-8"
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7 JOB_NAME="27_nslow" TEST_ARGS="--skip-slow" LINT=true USE_CACHE=true
+        - JOB="2.7" TEST_ARGS="--skip-slow" LINT=true
       addons:
         apt:
           packages:
           - python-gtk2
     - python: 3.5
       env:
-        - PYTHON_VERSION=3.5 JOB_NAME="35_nslow" TEST_ARGS="--skip-slow --skip-network" COVERAGE=true USE_CACHE=true
+        - JOB="3.5" TEST_ARGS="--skip-slow --skip-network" COVERAGE=true
       addons:
         apt:
           packages:
           - xsel
     - python: 3.6
       env:
-        - PYTHON_VERSION=3.6 JOB_NAME="36" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true USE_CACHE=true
+        - JOB="3.6" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true
       addons:
         apt:
           packages:
@@ -66,32 +66,32 @@ matrix:
     # In allow_failures
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7 JOB_NAME="27_slow" JOB_TAG="_SLOW" TEST_ARGS="--only-slow --skip-network" USE_CACHE=true
+        - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
     # In allow_failures
     - python: 2.7
       env:
-        - PYTHON_VERSION=2.7 JOB_NAME="27_build_test" JOB_TAG="_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true USE_CACHE=true
+        - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
     # In allow_failures
-    - python: 3.5
+    - python: 3.6
       env:
-        - PYTHON_VERSION=3.5 JOB_NAME="35_numpy_dev" JOB_TAG="_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" USE_CACHE=true
+        - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
     # In allow_failures
     - python: 3.5
       env:
-        - PYTHON_VERSION=3.5 JOB_NAME="doc_build" DOC_BUILD=true JOB_TAG="_DOC_BUILD" USE_CACHE=true
+        - JOB="3.5_DOC_BUILD" DOC_BUILD=true
     allow_failures:
       - python: 2.7
         env:
-          - PYTHON_VERSION=2.7 JOB_NAME="27_slow" JOB_TAG="_SLOW" TEST_ARGS="--only-slow --skip-network" USE_CACHE=true
+          - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
       - python: 2.7
         env:
-          - PYTHON_VERSION=2.7 JOB_NAME="27_build_test" JOB_TAG="_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true USE_CACHE=true
-      - python: 3.5
+          - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
+      - python: 3.6
         env:
-          - PYTHON_VERSION=3.5 JOB_NAME="35_numpy_dev" JOB_TAG="_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" USE_CACHE=true
+          - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
       - python: 3.5
         env:
-          - PYTHON_VERSION=3.5 JOB_NAME="doc_build" DOC_BUILD=true JOB_TAG="_DOC_BUILD" USE_CACHE=true
+          - JOB="3.5_DOC_BUILD" DOC_BUILD=true
 
 before_install:
   - echo "before_install"
@@ -107,7 +107,6 @@ before_install:
 
 install:
   - echo "install start"
-  - ci/check_cache.sh
   - ci/prep_cython_cache.sh
   - ci/install_travis.sh
   - ci/submit_cython_cache.sh

appveyor.yml

@@ -72,11 +72,11 @@ install:
   - cmd: conda info -a
 
   # create our env
-  - cmd: conda create -q -n pandas python=%PYTHON_VERSION% cython pytest
+  - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest
   - cmd: activate pandas
-  - SET REQ=ci\requirements-%PYTHON_VERSION%-%PYTHON_ARCH%.run
+  - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.run
   - cmd: echo "installing requirements from %REQ%"
-  - cmd: conda install -n pandas -q --file=%REQ%
+  - cmd: conda install -n pandas --file=%REQ%
   - cmd: conda list -n pandas
   - cmd: echo "installing requirements from %REQ% - done"
 

ci/check_cache.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+# currently not used
+# script to make sure that cache is clean
+# Travis CI now handles this
+
 if [ "$TRAVIS_PULL_REQUEST" == "false" ]
 then
     echo "Not a PR: checking for changes in ci/ from last 2 commits"

ci/install_circle.sh

@@ -46,9 +46,9 @@ echo "[environmental variable file]"
 cat $ENVS_FILE
 source $ENVS_FILE
 
-export REQ_BUILD=ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build
-export REQ_RUN=ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run
-export REQ_PIP=ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip
+export REQ_BUILD=ci/requirements-${JOB}.build
+export REQ_RUN=ci/requirements-${JOB}.run
+export REQ_PIP=ci/requirements-${JOB}.pip
 
 # edit the locale override if needed
 if [ -n "$LOCALE_OVERRIDE" ]; then
@@ -61,16 +61,13 @@ if [ -n "$LOCALE_OVERRIDE" ]; then
     echo
 fi
 
-# create new env
-echo "[create env]"
-time conda create -q -n pandas python=${PYTHON_VERSION} pytest || exit 1
+# create envbuild deps
+echo "[create env: ${REQ_BUILD}]"
+time conda create -n pandas -q --file=${REQ_BUILD} || exit 1
+time conda install -n pandas pytest || exit 1
 
 source activate pandas
 
-# build deps
-echo "[build installs: ${REQ_BUILD}]"
-time conda install -q --file=${REQ_BUILD} || exit 1
-
 # build but don't install
 echo "[build em]"
 time python setup.py build_ext --inplace || exit 1
@@ -84,5 +81,5 @@ fi
 # we may have additional pip installs
 echo "[pip installs: ${REQ_PIP}]"
 if [ -e ${REQ_PIP} ]; then
-   pip install -q -r $REQ_PIP
+   pip install -r $REQ_PIP
 fi

ci/install_travis.sh

@@ -68,15 +68,15 @@ conda info -a || exit 1
 
 # set the compiler cache to work
 echo
-if [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+if [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then
     echo "[Using ccache]"
     export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
     gcc=$(which gcc)
     echo "[gcc]: $gcc"
     ccache=$(which ccache)
     echo "[ccache]: $ccache"
     export CC='ccache gcc'
-elif [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+elif [ -z "$NOCACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then
     echo "[Using ccache]"
     time brew install ccache
     export PATH=/usr/local/opt/ccache/libexec:$PATH
@@ -91,35 +91,22 @@ fi
 echo
 echo "[create env]"
 
-# may have installation instructions for this build
-INSTALL="ci/install-${PYTHON_VERSION}${JOB_TAG}.sh"
-if [ -e ${INSTALL} ]; then
-    time bash $INSTALL || exit 1
-else
-    # create new env
-    # this may already exists, in which case our caching worked
-    time conda create -n pandas python=$PYTHON_VERSION pytest nomkl
-fi
+# create our environment
+REQ="ci/requirements-${JOB}.build"
+time conda create -n pandas --file=${REQ} || exit 1
 
-# build deps
-echo
-echo "[build installs]"
-REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build"
-if [ -e ${REQ} ]; then
-    time conda install -n pandas --file=${REQ} || exit 1
-fi
+source activate pandas
 
 # may have addtl installation instructions for this build
 echo
 echo "[build addtl installs]"
-REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build.sh"
+REQ="ci/requirements-${JOB}.build.sh"
 if [ -e ${REQ} ]; then
     time bash $REQ || exit 1
 fi
 
-source activate pandas
-
-pip install pytest-xdist
+time conda install -n pandas pytest
+time pip install pytest-xdist
 
 if [ "$LINT" ]; then
    conda install flake8
@@ -152,23 +139,23 @@ fi
 # we may have run installations
 echo
 echo "[conda installs]"
-REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run"
+REQ="ci/requirements-${JOB}.run"
 if [ -e ${REQ} ]; then
     time conda install -n pandas --file=${REQ} || exit 1
 fi
 
 # we may have additional pip installs
 echo
 echo "[pip installs]"
-REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip"
+REQ="ci/requirements-${JOB}.pip"
 if [ -e ${REQ} ]; then
    pip install -r $REQ
 fi
 
 # may have addtl installation instructions for this build
 echo
 echo "[addtl installs]"
-REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh"
+REQ="ci/requirements-${JOB}.sh"
 if [ -e ${REQ} ]; then
     time bash $REQ || exit 1
 fi

ci/prep_cython_cache.sh

@@ -22,7 +22,7 @@ fi
 
 home_dir=$(pwd)
 
-if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then
+if [ -f "$CACHE_File" ] && [ -z "$NOCACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then
 
     echo "Cache available - checking pyx diff"
 
@@ -57,16 +57,16 @@ if [ -f "$CACHE_File" ] && [ "$USE_CACHE" ] && [ -d "$PYX_CACHE_DIR" ]; then
 
 fi
 
-if [ $clear_cache -eq 0 ] && [ "$USE_CACHE" ]
+if [ $clear_cache -eq 0 ] && [ -z "$NOCACHE" ]
 then
-    # No and use_cache is set
+    # No and nocache is not set
     echo "Will reuse cached cython file"
     cd /
     tar xvmf $CACHE_File
     cd $home_dir
 else
     echo "Rebuilding cythonized files"
-    echo "Use cache (Blank if not set) = $USE_CACHE"
+    echo "No cache = $NOCACHE"
     echo "Clear cache (1=YES) = $clear_cache"
 fi
 

ci/requirements-2.7.build

@@ -1,4 +1,6 @@
+python=2.7*
 python-dateutil=2.4.1
 pytz=2013b
+nomkl
 numpy
 cython=0.23

ci/requirements-2.7_BUILD_TEST.build

@@ -1,4 +1,6 @@
+python=2.7*
 dateutil
 pytz
+nomkl
 numpy
 cython

ci/requirements-2.7_COMPAT.build

@@ -1,3 +1,4 @@
+python=2.7*
 numpy=1.7.1
 cython=0.23
 dateutil=1.5

ci/requirements-2.7_LOCALE.build

@@ -1,3 +1,4 @@
+python=2.7*
 python-dateutil
 pytz=2013b
 numpy=1.8.2

ci/requirements-2.7_SLOW.build

@@ -1,3 +1,4 @@
+python=2.7*
 python-dateutil
 pytz
 numpy=1.8.2

ci/requirements-2.7-64.run renamed to ci/requirements-2.7_WIN.run

@@ -1,3 +1,4 @@
+python=3.4*
 numpy=1.8.1
 cython=0.24.1
 libgfortran=1.0

ci/requirements-3.4-64.run

@@ -1,4 +1,6 @@
+python=3.4*
 python-dateutil
 pytz
+nomkl
 numpy=1.10*
 cython

ci/requirements-3.4.build

@@ -1,4 +1,6 @@
+python=3.5*
 python-dateutil
 pytz
+nomkl
 numpy=1.11.3
 cython

ci/requirements-3.4_SLOW.build

@@ -1,4 +1,6 @@
+python=3.5*
 python-dateutil
 pytz
+nomkl
 numpy
 cython

ci/requirements-3.5.build

@@ -1,4 +1,6 @@
+python=3.5*
 python-dateutil
 pytz
+nomkl
 numpy
 cython

ci/requirements-3.5_ASCII.build

@@ -1,2 +1,4 @@
+python=3.5*
+nomkl
 numpy=1.10.4
 cython

ci/requirements-3.5_DOC_BUILD.build

@@ -1,4 +1,6 @@
+python=3.6*
 python-dateutil
 pytz
+nomkl
 numpy
 cython

ci/requirements-3.5_OSX.build

@@ -1,3 +1,4 @@
+python=3.6*
 python-dateutil
 pytz
 cython

ci/requirements-3.6.build

@@ -21,13 +21,13 @@ dependencies:
     - >
       case $CIRCLE_NODE_INDEX in
        0)
-         sudo apt-get install language-pack-it && ./ci/install_circle.sh PYTHON_VERSION=2.7 JOB_TAG="_COMPAT" LOCALE_OVERRIDE="it_IT.UTF-8" ;;
+         sudo apt-get install language-pack-it && ./ci/install_circle.sh JOB="2.7_COMPAT" LOCALE_OVERRIDE="it_IT.UTF-8" ;;
        1)
-         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh PYTHON_VERSION=3.4 JOB_TAG="_SLOW" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
+         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.4_SLOW" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
        2)
-         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh PYTHON_VERSION=3.4 JOB_TAG="" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
+         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.4" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
        3)
-         ./ci/install_circle.sh PYTHON_VERSION=3.5 JOB_TAG="_ASCII" LOCALE_OVERRIDE="C" ;;
+         ./ci/install_circle.sh JOB="3.5_ASCII" LOCALE_OVERRIDE="C" ;;
       esac
     - ./ci/show_circle.sh