Merge pull request #7971 from cpcloud/conda

CI/WIP: Use conda for most deps

Author: cpcloud

.travis.yml

@@ -2,6 +2,7 @@
 language: python
 
 env:
+
   global:
   # scatterci API key
   #- secure: "Bx5umgo6WjuGY+5XFa004xjCiX/vq0CyMZ/ETzcs7EIBI1BE/0fIDXOoWhoxbY9HPfdPGlDnDgB9nGqr5wArO2s+BavyKBWg6osZ3dmkfuJPMOWeyCa92EeP+sfKw8e5HSU5MizW9e319wHWOF/xkzdHR7T67Qd5erhv91x4DnQ="
@@ -19,38 +20,44 @@ matrix:
       - NOSE_ARGS="not slow and not network and not disabled"
       - CLIPBOARD=xclip
       - LOCALE_OVERRIDE="it_IT.UTF-8"
+      - BUILD_TYPE=conda
       - JOB_NAME: "26_nslow_nnet"
     - python: 2.7
       env:
       - NOSE_ARGS="slow and not network and not disabled"
       - LOCALE_OVERRIDE="zh_CN.GB18030"
       - FULL_DEPS=true
       - JOB_TAG=_LOCALE
+      - BUILD_TYPE=conda
       - JOB_NAME: "27_slow_nnet_LOCALE"
     - python: 2.7
       env:
       - NOSE_ARGS="not slow and not disabled"
       - FULL_DEPS=true
       - CLIPBOARD_GUI=gtk2
+      - BUILD_TYPE=conda
       - JOB_NAME: "27_nslow"
       - DOC_BUILD=true # if rst files were changed, build docs in parallel with tests
     - python: 3.3
       env:
       - NOSE_ARGS="not slow and not disabled"
       - FULL_DEPS=true
       - CLIPBOARD=xsel
+      - BUILD_TYPE=conda
       - JOB_NAME: "33_nslow"
     - python: 3.4
       env:
       - NOSE_ARGS="not slow and not disabled"
       - FULL_DEPS=true
       - CLIPBOARD=xsel
+      - BUILD_TYPE=conda
       - JOB_NAME: "34_nslow"
     - python: 3.2
       env:
       - NOSE_ARGS="not slow and not disabled"
       - FULL_DEPS=true
       - CLIPBOARD_GUI=qt4
+      - BUILD_TYPE=pydata
       - JOB_NAME: "32_nslow"
     - python: 2.7
       env:
@@ -59,13 +66,15 @@ matrix:
       - JOB_NAME: "27_numpy_master"
       - JOB_TAG=_NUMPY_DEV_master
       - NUMPY_BUILD=master
+      - BUILD_TYPE=pydata
       - PANDAS_TESTING_MODE="deprecate"
     allow_failures:
       - python: 3.2
         env:
         - NOSE_ARGS="not slow and not disabled"
         - FULL_DEPS=true
         - CLIPBOARD_GUI=qt4
+        - BUILD_TYPE=pydata
         - JOB_NAME: "32_nslow"
       - python: 2.7
         env:
@@ -74,11 +83,14 @@ matrix:
         - JOB_NAME: "27_numpy_master"
         - JOB_TAG=_NUMPY_DEV_master
         - NUMPY_BUILD=master
+        - BUILD_TYPE=pydata
         - PANDAS_TESTING_MODE="deprecate"
 
 before_install:
   - echo "before_install"
   - echo $VIRTUAL_ENV
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - sudo apt-get install ccache
   - df -h
   - date
   - pwd
@@ -92,7 +104,7 @@ before_install:
 install:
   - echo "install"
   - ci/prep_ccache.sh
-  - ci/install.sh
+  - ci/install_${BUILD_TYPE}.sh
   - ci/submit_ccache.sh
 
 before_script:
@@ -106,6 +118,6 @@ script:
 
 after_script:
   - if [ -f /tmp/doc.log ]; then cat /tmp/doc.log; fi
-  - ci/print_versions.py
+  - source activate pandas && ci/print_versions.py
   - ci/print_skipped.py /tmp/nosetests.xml
   - ci/after_script.sh

ci/install_conda.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# There are 2 distinct pieces that get zipped and cached
+# - The venv site-packages dir including the installed dependencies
+# - The pandas build artifacts, using the build cache support via
+#   scripts/use_build_cache.py
+#
+# if the user opted in to use the cache and we're on a whitelisted fork
+# - if the server doesn't hold a cached version of venv/pandas build,
+#   do things the slow way, and put the results on the cache server
+#   for the next time.
+# -  if the cache files are available, instal some necessaries via apt
+#    (no compiling needed), then directly goto script and collect 200$.
+#
+
+function edit_init()
+{
+    if [ -n "$LOCALE_OVERRIDE" ]; then
+        echo "Adding locale to the first line of pandas/__init__.py"
+        rm -f pandas/__init__.pyc
+        sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n"
+        sed -i "$sedc" pandas/__init__.py
+        echo "head -4 pandas/__init__.py"
+        head -4 pandas/__init__.py
+        echo
+    fi
+}
+
+edit_init
+
+python_major_version="${TRAVIS_PYTHON_VERSION:0:1}"
+[ "$python_major_version" == "2" ] && python_major_version=""
+
+home_dir=$(pwd)
+echo "home_dir: [$home_dir]"
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    # make sure the locale is available
+    # probably useless, since you would need to relogin
+    time sudo locale-gen "$LOCALE_OVERRIDE"
+fi
+
+# Need to enable for locale testing. The location of the locale file(s) is
+# distro specific. For example, on Arch Linux all of the locales are in a
+# commented file--/etc/locale.gen--that must be commented in to be used
+# whereas Ubuntu looks in /var/lib/locales/supported.d/* and generates locales
+# based on what's in the files in that folder
+time echo 'it_CH.UTF-8 UTF-8' | sudo tee -a /var/lib/locales/supported.d/it
+time sudo locale-gen
+
+
+# install gui for clipboard testing
+if [ -n "$CLIPBOARD_GUI" ]; then
+    echo "Using CLIPBOARD_GUI: $CLIPBOARD_GUI"
+    [ -n "$python_major_version" ] && py="py"
+    python_cb_gui_pkg=python${python_major_version}-${py}${CLIPBOARD_GUI}
+    time sudo apt-get $APT_ARGS install $python_cb_gui_pkg
+fi
+
+
+# install a clipboard if $CLIPBOARD is not empty
+if [ -n "$CLIPBOARD" ]; then
+    echo "Using clipboard: $CLIPBOARD"
+    time sudo apt-get $APT_ARGS install $CLIPBOARD
+fi
+
+python_major_version="${TRAVIS_PYTHON_VERSION:0:1}"
+[ "$python_major_version" == "2" ] && python_major_version=""
+
+wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh || exit 1
+bash miniconda.sh -b -p $HOME/miniconda || exit 1
+
+conda config --set always_yes yes --set changeps1 no || exit 1
+conda update -q conda || exit 1
+conda config --add channels http://conda.binstar.org/pandas || exit 1
+
+# Useful for debugging any issues with conda
+conda info -a || exit 1
+
+conda create -n pandas python=$TRAVIS_PYTHON_VERSION || exit 1
+conda install -n pandas --file=ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.txt || exit 1
+
+conda install -n pandas pip setuptools nose || exit 1
+conda remove -n pandas pandas
+
+source activate pandas
+
+# set the compiler cache to work
+if [ "$IRON_TOKEN" ]; then
+    export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
+    gcc=$(which gcc)
+    echo "gcc: $gcc"
+    ccache=$(which ccache)
+    echo "ccache: $ccache"
+    export CC='ccache gcc'
+fi
+
+python setup.py build_ext --inplace && python setup.py develop
+
+for package in beautifulsoup4 'python-dateutil'; do
+    pip uninstall --yes $package
+done
+
+true

ci/install.sh renamed to ci/install_pydata.sh

@@ -137,11 +137,8 @@ if [ "$IRON_TOKEN" ]; then
 fi
 
 # build pandas
-time python setup.py sdist
-pip uninstall cython -y
-
-# install pandas
-time pip install $(find dist | grep gz | head -n 1)
+python setup.py build_ext --inplace
+python setup.py develop
 
 # restore cython (if not numpy building)
 if [ -z "$NUMPY_BUILD" ]; then

ci/requirements-2.6.txt

@@ -1,16 +1,16 @@
-numpy==1.7.0
-cython==0.19.1
-python-dateutil==1.5
-pytz==2013b
-http://www.crummy.com/software/BeautifulSoup/bs4/download/4.2/beautifulsoup4-4.2.0.tar.gz
-html5lib==1.0b2
-numexpr==1.4.2
-sqlalchemy==0.7.10
-pymysql==0.6.0
-psycopg2==2.5
-scipy==0.11.0
-statsmodels==0.4.3
-xlwt==0.7.5
-openpyxl==2.0.3
-xlsxwriter==0.4.6
-xlrd==0.9.2
+numpy=1.7.0
+cython=0.19.1
+dateutil=1.5
+pytz=2013b
+scipy=0.11.0
+xlwt=0.7.5
+xlrd=0.9.2
+openpyxl=2.0.3
+statsmodels=0.4.3
+html5lib=1.0b2
+beautiful-soup=4.2.0
+psycopg2=2.5.1
+numexpr=1.4.2
+pymysql=0.6.0
+sqlalchemy=0.7.8
+xlsxwriter=0.4.6

ci/requirements-2.7.txt

@@ -1,25 +1,25 @@
-python-dateutil==2.1
-pytz==2013b
-xlwt==0.7.5
-numpy==1.8.1
-cython==0.19.1
-bottleneck==0.6.0
-numexpr==2.2.2
-tables==3.0.0
-matplotlib==1.3.1
-openpyxl==1.6.2
-xlsxwriter==0.4.6
-xlrd==0.9.2
-patsy==0.1.0
-sqlalchemy==0.9.6
-pymysql==0.6.1
-psycopg2==2.5.2
-html5lib==1.0b2
-lxml==3.2.1
-scipy==0.13.3
-beautifulsoup4==4.2.1
-statsmodels==0.5.0
-boto==2.26.1
-httplib2==0.8
-python-gflags==2.0
-google-api-python-client==1.2
+dateutil=2.1
+pytz=2013b
+xlwt=0.7.5
+numpy=1.7.0
+cython=0.19.1
+numexpr=2.2.2
+pytables=3.0.0
+matplotlib=1.3.1
+openpyxl=1.6.2
+xlrd=0.9.2
+sqlalchemy=0.9.6
+lxml=3.2.1
+scipy
+xlsxwriter=0.4.6
+statsmodels
+boto=2.26.1
+bottleneck=0.8.0
+psycopg2=2.5.2
+patsy
+pymysql=0.6.1
+html5lib=1.0b2
+beautiful-soup=4.2.1
+httplib2=0.8
+python-gflags=2.0
+google-api-python-client=1.2

ci/requirements-2.7_LOCALE.txt

@@ -1,18 +1,18 @@
-python-dateutil
-pytz==2013b
-xlwt==0.7.5
-openpyxl==1.6.2
-xlsxwriter==0.4.6
-xlrd==0.9.2
-numpy==1.7.1
-cython==0.19.1
-bottleneck==0.6.0
-matplotlib==1.3.0
-patsy==0.1.0
-sqlalchemy==0.8.1
-html5lib==1.0b2
-lxml==3.2.1
-scipy==0.10.0
-beautifulsoup4==4.2.1
-statsmodels==0.4.3
-bigquery==2.0.17
+dateutil
+pytz=2013b
+xlwt=0.7.5
+openpyxl=1.6.2
+xlsxwriter=0.4.6
+xlrd=0.9.2
+numpy=1.7.1
+cython=0.19.1
+bottleneck=0.8.0
+matplotlib=1.3.0
+patsy=0.1.0
+sqlalchemy=0.8.1
+html5lib=1.0b2
+lxml=3.2.1
+scipy=0.11.0
+beautiful-soup=4.2.1
+statsmodels=0.4.3
+bigquery=2.0.17

ci/requirements-3.3.txt

@@ -1,17 +1,17 @@
-python-dateutil==2.2
-pytz==2013b
-openpyxl==1.6.2
-xlsxwriter==0.4.6
-xlrd==0.9.2
-html5lib==1.0b2
-numpy==1.8.0
-cython==0.19.1
-numexpr==2.3
-tables==3.1.0
-bottleneck==0.8.0
-matplotlib==1.2.1
-patsy==0.1.0
-lxml==3.2.1
-scipy==0.13.3
-beautifulsoup4==4.2.1
-statsmodels==0.5.0
+dateutil
+pytz=2013b
+openpyxl=1.6.2
+xlsxwriter=0.4.6
+xlrd=0.9.2
+html5lib=1.0b2
+numpy=1.8.0
+cython=0.19.1
+numexpr
+pytables
+bottleneck=0.8.0
+matplotlib
+patsy
+lxml=3.2.1
+scipy
+beautiful-soup=4.2.1
+statsmodels